2 * Huawei HiNIC PCI Express Linux driver
3 * Copyright(c) 2017 Huawei Technologies Co., Ltd
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 #include <linux/kernel.h>
17 #include <linux/types.h>
18 #include <linux/pci.h>
19 #include <linux/device.h>
20 #include <linux/errno.h>
21 #include <linux/slab.h>
22 #include <linux/semaphore.h>
23 #include <linux/dma-mapping.h>
25 #include <linux/err.h>
27 #include "hinic_hw_if.h"
28 #include "hinic_hw_eqs.h"
29 #include "hinic_hw_wqe.h"
30 #include "hinic_hw_wq.h"
31 #include "hinic_hw_cmdq.h"
32 #include "hinic_hw_qp_ctxt.h"
33 #include "hinic_hw_qp.h"
34 #include "hinic_hw_io.h"
36 #define CI_Q_ADDR_SIZE sizeof(u32)
38 #define CI_ADDR(base_addr, q_id) ((base_addr) + \
39 (q_id) * CI_Q_ADDR_SIZE)
41 #define CI_TABLE_SIZE(num_qps) ((num_qps) * CI_Q_ADDR_SIZE)
43 #define DB_IDX(db, db_base) \
44 (((unsigned long)(db) - (unsigned long)(db_base)) / HINIC_DB_PAGE_SIZE)
47 IO_CMD_MODIFY_QUEUE_CTXT = 0,
50 static void init_db_area_idx(struct hinic_free_db_area *free_db_area)
54 for (i = 0; i < HINIC_DB_MAX_AREAS; i++)
55 free_db_area->db_idx[i] = i;
57 free_db_area->alloc_pos = 0;
58 free_db_area->return_pos = HINIC_DB_MAX_AREAS;
60 free_db_area->num_free = HINIC_DB_MAX_AREAS;
62 sema_init(&free_db_area->idx_lock, 1);
65 static void __iomem *get_db_area(struct hinic_func_to_io *func_to_io)
67 struct hinic_free_db_area *free_db_area = &func_to_io->free_db_area;
70 down(&free_db_area->idx_lock);
72 free_db_area->num_free--;
74 if (free_db_area->num_free < 0) {
75 free_db_area->num_free++;
76 up(&free_db_area->idx_lock);
77 return ERR_PTR(-ENOMEM);
80 pos = free_db_area->alloc_pos++;
81 pos &= HINIC_DB_MAX_AREAS - 1;
83 idx = free_db_area->db_idx[pos];
85 free_db_area->db_idx[pos] = -1;
87 up(&free_db_area->idx_lock);
89 return func_to_io->db_base + idx * HINIC_DB_PAGE_SIZE;
92 static void return_db_area(struct hinic_func_to_io *func_to_io,
93 void __iomem *db_base)
95 struct hinic_free_db_area *free_db_area = &func_to_io->free_db_area;
96 int pos, idx = DB_IDX(db_base, func_to_io->db_base);
98 down(&free_db_area->idx_lock);
100 pos = free_db_area->return_pos++;
101 pos &= HINIC_DB_MAX_AREAS - 1;
103 free_db_area->db_idx[pos] = idx;
105 free_db_area->num_free++;
107 up(&free_db_area->idx_lock);
110 static int write_sq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
113 struct hinic_hwif *hwif = func_to_io->hwif;
114 struct hinic_sq_ctxt_block *sq_ctxt_block;
115 struct pci_dev *pdev = hwif->pdev;
116 struct hinic_cmdq_buf cmdq_buf;
117 struct hinic_sq_ctxt *sq_ctxt;
122 err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
124 dev_err(&pdev->dev, "Failed to allocate cmdq buf\n");
128 sq_ctxt_block = cmdq_buf.buf;
129 sq_ctxt = sq_ctxt_block->sq_ctxt;
131 hinic_qp_prepare_header(&sq_ctxt_block->hdr, HINIC_QP_CTXT_TYPE_SQ,
132 num_sqs, func_to_io->max_qps);
133 for (i = 0; i < num_sqs; i++) {
134 qp = &func_to_io->qps[i];
136 hinic_sq_prepare_ctxt(&sq_ctxt[i], &qp->sq,
137 base_qpn + qp->q_id);
140 cmdq_buf.size = HINIC_SQ_CTXT_SIZE(num_sqs);
142 err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC,
143 IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf,
145 if ((err) || (out_param != 0)) {
146 dev_err(&pdev->dev, "Failed to set SQ ctxts\n");
150 hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
154 static int write_rq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
157 struct hinic_hwif *hwif = func_to_io->hwif;
158 struct hinic_rq_ctxt_block *rq_ctxt_block;
159 struct pci_dev *pdev = hwif->pdev;
160 struct hinic_cmdq_buf cmdq_buf;
161 struct hinic_rq_ctxt *rq_ctxt;
166 err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
168 dev_err(&pdev->dev, "Failed to allocate cmdq buf\n");
172 rq_ctxt_block = cmdq_buf.buf;
173 rq_ctxt = rq_ctxt_block->rq_ctxt;
175 hinic_qp_prepare_header(&rq_ctxt_block->hdr, HINIC_QP_CTXT_TYPE_RQ,
176 num_rqs, func_to_io->max_qps);
177 for (i = 0; i < num_rqs; i++) {
178 qp = &func_to_io->qps[i];
180 hinic_rq_prepare_ctxt(&rq_ctxt[i], &qp->rq,
181 base_qpn + qp->q_id);
184 cmdq_buf.size = HINIC_RQ_CTXT_SIZE(num_rqs);
186 err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC,
187 IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf,
189 if ((err) || (out_param != 0)) {
190 dev_err(&pdev->dev, "Failed to set RQ ctxts\n");
194 hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
199 * write_qp_ctxts - write the qp ctxt to HW
200 * @func_to_io: func to io channel that holds the IO components
201 * @base_qpn: first qp number
202 * @num_qps: number of qps to write
204 * Return 0 - Success, negative - Failure
206 static int write_qp_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
209 return (write_sq_ctxts(func_to_io, base_qpn, num_qps) ||
210 write_rq_ctxts(func_to_io, base_qpn, num_qps));
214 * init_qp - Initialize a Queue Pair
215 * @func_to_io: func to io channel that holds the IO components
216 * @qp: pointer to the qp to initialize
217 * @q_id: the id of the qp
218 * @sq_msix_entry: msix entry for sq
219 * @rq_msix_entry: msix entry for rq
221 * Return 0 - Success, negative - Failure
223 static int init_qp(struct hinic_func_to_io *func_to_io,
224 struct hinic_qp *qp, int q_id,
225 struct msix_entry *sq_msix_entry,
226 struct msix_entry *rq_msix_entry)
228 struct hinic_hwif *hwif = func_to_io->hwif;
229 struct pci_dev *pdev = hwif->pdev;
230 void __iomem *db_base;
235 err = hinic_wq_allocate(&func_to_io->wqs, &func_to_io->sq_wq[q_id],
236 HINIC_SQ_WQEBB_SIZE, HINIC_SQ_PAGE_SIZE,
237 HINIC_SQ_DEPTH, HINIC_SQ_WQE_MAX_SIZE);
239 dev_err(&pdev->dev, "Failed to allocate WQ for SQ\n");
243 err = hinic_wq_allocate(&func_to_io->wqs, &func_to_io->rq_wq[q_id],
244 HINIC_RQ_WQEBB_SIZE, HINIC_RQ_PAGE_SIZE,
245 HINIC_RQ_DEPTH, HINIC_RQ_WQE_SIZE);
247 dev_err(&pdev->dev, "Failed to allocate WQ for RQ\n");
251 db_base = get_db_area(func_to_io);
252 if (IS_ERR(db_base)) {
253 dev_err(&pdev->dev, "Failed to get DB area for SQ\n");
254 err = PTR_ERR(db_base);
258 func_to_io->sq_db[q_id] = db_base;
260 err = hinic_init_sq(&qp->sq, hwif, &func_to_io->sq_wq[q_id],
262 CI_ADDR(func_to_io->ci_addr_base, q_id),
263 CI_ADDR(func_to_io->ci_dma_base, q_id), db_base);
265 dev_err(&pdev->dev, "Failed to init SQ\n");
269 err = hinic_init_rq(&qp->rq, hwif, &func_to_io->rq_wq[q_id],
272 dev_err(&pdev->dev, "Failed to init RQ\n");
279 hinic_clean_sq(&qp->sq);
282 return_db_area(func_to_io, db_base);
285 hinic_wq_free(&func_to_io->wqs, &func_to_io->rq_wq[q_id]);
288 hinic_wq_free(&func_to_io->wqs, &func_to_io->sq_wq[q_id]);
293 * destroy_qp - Clean the resources of a Queue Pair
294 * @func_to_io: func to io channel that holds the IO components
295 * @qp: pointer to the qp to clean
297 static void destroy_qp(struct hinic_func_to_io *func_to_io,
302 hinic_clean_rq(&qp->rq);
303 hinic_clean_sq(&qp->sq);
305 return_db_area(func_to_io, func_to_io->sq_db[q_id]);
307 hinic_wq_free(&func_to_io->wqs, &func_to_io->rq_wq[q_id]);
308 hinic_wq_free(&func_to_io->wqs, &func_to_io->sq_wq[q_id]);
312 * hinic_io_create_qps - Create Queue Pairs
313 * @func_to_io: func to io channel that holds the IO components
314 * @base_qpn: base qp number
315 * @num_qps: number queue pairs to create
316 * @sq_msix_entry: msix entries for sq
317 * @rq_msix_entry: msix entries for rq
319 * Return 0 - Success, negative - Failure
321 int hinic_io_create_qps(struct hinic_func_to_io *func_to_io,
322 u16 base_qpn, int num_qps,
323 struct msix_entry *sq_msix_entries,
324 struct msix_entry *rq_msix_entries)
326 struct hinic_hwif *hwif = func_to_io->hwif;
327 struct pci_dev *pdev = hwif->pdev;
328 size_t qps_size, wq_size, db_size;
332 qps_size = num_qps * sizeof(*func_to_io->qps);
333 func_to_io->qps = devm_kzalloc(&pdev->dev, qps_size, GFP_KERNEL);
334 if (!func_to_io->qps)
337 wq_size = num_qps * sizeof(*func_to_io->sq_wq);
338 func_to_io->sq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL);
339 if (!func_to_io->sq_wq) {
344 wq_size = num_qps * sizeof(*func_to_io->rq_wq);
345 func_to_io->rq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL);
346 if (!func_to_io->rq_wq) {
351 db_size = num_qps * sizeof(*func_to_io->sq_db);
352 func_to_io->sq_db = devm_kzalloc(&pdev->dev, db_size, GFP_KERNEL);
353 if (!func_to_io->sq_db) {
358 ci_addr_base = dma_alloc_coherent(&pdev->dev, CI_TABLE_SIZE(num_qps),
359 &func_to_io->ci_dma_base,
362 dev_err(&pdev->dev, "Failed to allocate CI area\n");
367 func_to_io->ci_addr_base = ci_addr_base;
369 for (i = 0; i < num_qps; i++) {
370 err = init_qp(func_to_io, &func_to_io->qps[i], i,
371 &sq_msix_entries[i], &rq_msix_entries[i]);
373 dev_err(&pdev->dev, "Failed to create QP %d\n", i);
378 err = write_qp_ctxts(func_to_io, base_qpn, num_qps);
380 dev_err(&pdev->dev, "Failed to init QP ctxts\n");
381 goto err_write_qp_ctxts;
388 for (j = 0; j < i; j++)
389 destroy_qp(func_to_io, &func_to_io->qps[j]);
391 dma_free_coherent(&pdev->dev, CI_TABLE_SIZE(num_qps),
392 func_to_io->ci_addr_base, func_to_io->ci_dma_base);
395 devm_kfree(&pdev->dev, func_to_io->sq_db);
398 devm_kfree(&pdev->dev, func_to_io->rq_wq);
401 devm_kfree(&pdev->dev, func_to_io->sq_wq);
404 devm_kfree(&pdev->dev, func_to_io->qps);
409 * hinic_io_destroy_qps - Destroy the IO Queue Pairs
410 * @func_to_io: func to io channel that holds the IO components
411 * @num_qps: number queue pairs to destroy
413 void hinic_io_destroy_qps(struct hinic_func_to_io *func_to_io, int num_qps)
415 struct hinic_hwif *hwif = func_to_io->hwif;
416 struct pci_dev *pdev = hwif->pdev;
417 size_t ci_table_size;
420 ci_table_size = CI_TABLE_SIZE(num_qps);
422 for (i = 0; i < num_qps; i++)
423 destroy_qp(func_to_io, &func_to_io->qps[i]);
425 dma_free_coherent(&pdev->dev, ci_table_size, func_to_io->ci_addr_base,
426 func_to_io->ci_dma_base);
428 devm_kfree(&pdev->dev, func_to_io->sq_db);
430 devm_kfree(&pdev->dev, func_to_io->rq_wq);
431 devm_kfree(&pdev->dev, func_to_io->sq_wq);
433 devm_kfree(&pdev->dev, func_to_io->qps);
437 * hinic_io_init - Initialize the IO components
438 * @func_to_io: func to io channel that holds the IO components
439 * @hwif: HW interface for accessing IO
440 * @max_qps: maximum QPs in HW
441 * @num_ceqs: number completion event queues
442 * @ceq_msix_entries: msix entries for ceqs
444 * Return 0 - Success, negative - Failure
446 int hinic_io_init(struct hinic_func_to_io *func_to_io,
447 struct hinic_hwif *hwif, u16 max_qps, int num_ceqs,
448 struct msix_entry *ceq_msix_entries)
450 struct pci_dev *pdev = hwif->pdev;
451 enum hinic_cmdq_type cmdq, type;
452 void __iomem *db_area;
455 func_to_io->hwif = hwif;
456 func_to_io->qps = NULL;
457 func_to_io->max_qps = max_qps;
459 err = hinic_ceqs_init(&func_to_io->ceqs, hwif, num_ceqs,
460 HINIC_DEFAULT_CEQ_LEN, HINIC_EQ_PAGE_SIZE,
463 dev_err(&pdev->dev, "Failed to init CEQs\n");
467 err = hinic_wqs_alloc(&func_to_io->wqs, 2 * max_qps, hwif);
469 dev_err(&pdev->dev, "Failed to allocate WQS for IO\n");
473 func_to_io->db_base = pci_ioremap_bar(pdev, HINIC_PCI_DB_BAR);
474 if (!func_to_io->db_base) {
475 dev_err(&pdev->dev, "Failed to remap IO DB area\n");
480 init_db_area_idx(&func_to_io->free_db_area);
482 for (cmdq = HINIC_CMDQ_SYNC; cmdq < HINIC_MAX_CMDQ_TYPES; cmdq++) {
483 db_area = get_db_area(func_to_io);
484 if (IS_ERR(db_area)) {
485 dev_err(&pdev->dev, "Failed to get cmdq db area\n");
486 err = PTR_ERR(db_area);
490 func_to_io->cmdq_db_area[cmdq] = db_area;
493 err = hinic_init_cmdqs(&func_to_io->cmdqs, hwif,
494 func_to_io->cmdq_db_area);
496 dev_err(&pdev->dev, "Failed to initialize cmdqs\n");
504 for (type = HINIC_CMDQ_SYNC; type < cmdq; type++)
505 return_db_area(func_to_io, func_to_io->cmdq_db_area[type]);
507 iounmap(func_to_io->db_base);
510 hinic_wqs_free(&func_to_io->wqs);
513 hinic_ceqs_free(&func_to_io->ceqs);
518 * hinic_io_free - Free the IO components
519 * @func_to_io: func to io channel that holds the IO components
521 void hinic_io_free(struct hinic_func_to_io *func_to_io)
523 enum hinic_cmdq_type cmdq;
525 hinic_free_cmdqs(&func_to_io->cmdqs);
527 for (cmdq = HINIC_CMDQ_SYNC; cmdq < HINIC_MAX_CMDQ_TYPES; cmdq++)
528 return_db_area(func_to_io, func_to_io->cmdq_db_area[cmdq]);
530 iounmap(func_to_io->db_base);
531 hinic_wqs_free(&func_to_io->wqs);
532 hinic_ceqs_free(&func_to_io->ceqs);