drivers/infiniband/hw/bnxt_re/qplib_fp.c

   1 /*
   2  * Broadcom NetXtreme-E RoCE driver.
   3  *
   4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
   5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * BSD license below:
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  *
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in
  21  *    the documentation and/or other materials provided with the
  22  *    distribution.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
  28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Description: Fast Path Operators
  37  */
  38
  39 #include <linux/interrupt.h>
  40 #include <linux/spinlock.h>
  41 #include <linux/sched.h>
  42 #include <linux/slab.h>
  43 #include <linux/pci.h>
  44 #include <linux/prefetch.h>
  45
  46 #include "roce_hsi.h"
  47
  48 #include "qplib_res.h"
  49 #include "qplib_rcfw.h"
  50 #include "qplib_sp.h"
  51 #include "qplib_fp.h"
  52
  53 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
  54
  55 static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res,
  56                                        struct bnxt_qplib_qp *qp)
  57 {
  58         struct bnxt_qplib_q *rq = &qp->rq;
  59         struct bnxt_qplib_q *sq = &qp->sq;
  60
  61         if (qp->rq_hdr_buf)
  62                 dma_free_coherent(&res->pdev->dev,
  63                                   rq->hwq.max_elements * qp->rq_hdr_buf_size,
  64                                   qp->rq_hdr_buf, qp->rq_hdr_buf_map);
  65         if (qp->sq_hdr_buf)
  66                 dma_free_coherent(&res->pdev->dev,
  67                                   sq->hwq.max_elements * qp->sq_hdr_buf_size,
  68                                   qp->sq_hdr_buf, qp->sq_hdr_buf_map);
  69         qp->rq_hdr_buf = NULL;
  70         qp->sq_hdr_buf = NULL;
  71         qp->rq_hdr_buf_map = 0;
  72         qp->sq_hdr_buf_map = 0;
  73         qp->sq_hdr_buf_size = 0;
  74         qp->rq_hdr_buf_size = 0;
  75 }
  76
  77 static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res,
  78                                        struct bnxt_qplib_qp *qp)
  79 {
  80         struct bnxt_qplib_q *rq = &qp->rq;
  81         struct bnxt_qplib_q *sq = &qp->rq;
  82         int rc = 0;
  83
  84         if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {
  85                 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
  86                                         sq->hwq.max_elements *
  87                                         qp->sq_hdr_buf_size,
  88                                         &qp->sq_hdr_buf_map, GFP_KERNEL);
  89                 if (!qp->sq_hdr_buf) {
  90                         rc = -ENOMEM;
  91                         dev_err(&res->pdev->dev,
  92                                 "QPLIB: Failed to create sq_hdr_buf");
  93                         goto fail;
  94                 }
  95         }
  96
  97         if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {
  98                 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
  99                                                     rq->hwq.max_elements *
 100                                                     qp->rq_hdr_buf_size,
 101                                                     &qp->rq_hdr_buf_map,
 102                                                     GFP_KERNEL);
 103                 if (!qp->rq_hdr_buf) {
 104                         rc = -ENOMEM;
 105                         dev_err(&res->pdev->dev,
 106                                 "QPLIB: Failed to create rq_hdr_buf");
 107                         goto fail;
 108                 }
 109         }
 110         return 0;
 111
 112 fail:
 113         bnxt_qplib_free_qp_hdr_buf(res, qp);
 114         return rc;
 115 }
 116
 117 static void bnxt_qplib_service_nq(unsigned long data)
 118 {
 119         struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
 120         struct bnxt_qplib_hwq *hwq = &nq->hwq;
 121         struct nq_base *nqe, **nq_ptr;
 122         int num_cqne_processed = 0;
 123         u32 sw_cons, raw_cons;
 124         u16 type;
 125         int budget = nq->budget;
 126         u64 q_handle;
 127
 128         /* Service the NQ until empty */
 129         raw_cons = hwq->cons;
 130         while (budget--) {
 131                 sw_cons = HWQ_CMP(raw_cons, hwq);
 132                 nq_ptr = (struct nq_base **)hwq->pbl_ptr;
 133                 nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];
 134                 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
 135                         break;
 136
 137                 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
 138                 switch (type) {
 139                 case NQ_BASE_TYPE_CQ_NOTIFICATION:
 140                 {
 141                         struct nq_cn *nqcne = (struct nq_cn *)nqe;
 142
 143                         q_handle = le32_to_cpu(nqcne->cq_handle_low);
 144                         q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
 145                                                      << 32;
 146                         bnxt_qplib_arm_cq_enable((struct bnxt_qplib_cq *)
 147                                                  ((unsigned long)q_handle));
 148                         if (!nq->cqn_handler(nq, (struct bnxt_qplib_cq *)
 149                                                  ((unsigned long)q_handle)))
 150                                 num_cqne_processed++;
 151                         else
 152                                 dev_warn(&nq->pdev->dev,
 153                                          "QPLIB: cqn - type 0x%x not handled",
 154                                          type);
 155                         break;
 156                 }
 157                 case NQ_BASE_TYPE_DBQ_EVENT:
 158                         break;
 159                 default:
 160                         dev_warn(&nq->pdev->dev,
 161                                  "QPLIB: nqe with type = 0x%x not handled",
 162                                  type);
 163                         break;
 164                 }
 165                 raw_cons++;
 166         }
 167         if (hwq->cons != raw_cons) {
 168                 hwq->cons = raw_cons;
 169                 NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements);
 170         }
 171 }
 172
 173 static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
 174 {
 175         struct bnxt_qplib_nq *nq = dev_instance;
 176         struct bnxt_qplib_hwq *hwq = &nq->hwq;
 177         struct nq_base **nq_ptr;
 178         u32 sw_cons;
 179
 180         /* Prefetch the NQ element */
 181         sw_cons = HWQ_CMP(hwq->cons, hwq);
 182         nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr;
 183         prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);
 184
 185         /* Fan out to CPU affinitized kthreads? */
 186         tasklet_schedule(&nq->worker);
 187
 188         return IRQ_HANDLED;
 189 }
 190
 191 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
 192 {
 193         /* Make sure the HW is stopped! */
 194         synchronize_irq(nq->vector);
 195         tasklet_disable(&nq->worker);
 196         tasklet_kill(&nq->worker);
 197
 198         if (nq->requested) {
 199                 free_irq(nq->vector, nq);
 200                 nq->requested = false;
 201         }
 202         if (nq->bar_reg_iomem)
 203                 iounmap(nq->bar_reg_iomem);
 204         nq->bar_reg_iomem = NULL;
 205
 206         nq->cqn_handler = NULL;
 207         nq->srqn_handler = NULL;
 208         nq->vector = 0;
 209 }
 210
 211 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
 212                          int msix_vector, int bar_reg_offset,
 213                          int (*cqn_handler)(struct bnxt_qplib_nq *nq,
 214                                             struct bnxt_qplib_cq *),
 215                          int (*srqn_handler)(struct bnxt_qplib_nq *nq,
 216                                              void *, u8 event))
 217 {
 218         resource_size_t nq_base;
 219         int rc;
 220
 221         nq->pdev = pdev;
 222         nq->vector = msix_vector;
 223
 224         nq->cqn_handler = cqn_handler;
 225
 226         nq->srqn_handler = srqn_handler;
 227
 228         tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq);
 229
 230         nq->requested = false;
 231         rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, "bnxt_qplib_nq", nq);
 232         if (rc) {
 233                 dev_err(&nq->pdev->dev,
 234                         "Failed to request IRQ for NQ: %#x", rc);
 235                 bnxt_qplib_disable_nq(nq);
 236                 goto fail;
 237         }
 238         nq->requested = true;
 239         nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
 240         nq->bar_reg_off = bar_reg_offset;
 241         nq_base = pci_resource_start(pdev, nq->bar_reg);
 242         if (!nq_base) {
 243                 rc = -ENOMEM;
 244                 goto fail;
 245         }
 246         nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4);
 247         if (!nq->bar_reg_iomem) {
 248                 rc = -ENOMEM;
 249                 goto fail;
 250         }
 251         NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
 252
 253         return 0;
 254 fail:
 255         bnxt_qplib_disable_nq(nq);
 256         return rc;
 257 }
 258
 259 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
 260 {
 261         if (nq->hwq.max_elements)
 262                 bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);
 263 }
 264
 265 int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
 266 {
 267         nq->pdev = pdev;
 268         if (!nq->hwq.max_elements ||
 269             nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
 270                 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
 271
 272         if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0,
 273                                       &nq->hwq.max_elements,
 274                                       BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,
 275                                       PAGE_SIZE, HWQ_TYPE_L2_CMPL))
 276                 return -ENOMEM;
 277
 278         nq->budget = 8;
 279         return 0;
 280 }
 281
 282 /* QP */
 283 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 284 {
 285         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 286         struct cmdq_create_qp1 req;
 287         struct creq_create_qp1_resp resp;
 288         struct bnxt_qplib_pbl *pbl;
 289         struct bnxt_qplib_q *sq = &qp->sq;
 290         struct bnxt_qplib_q *rq = &qp->rq;
 291         int rc;
 292         u16 cmd_flags = 0;
 293         u32 qp_flags = 0;
 294
 295         RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
 296
 297         /* General */
 298         req.type = qp->type;
 299         req.dpi = cpu_to_le32(qp->dpi->dpi);
 300         req.qp_handle = cpu_to_le64(qp->qp_handle);
 301
 302         /* SQ */
 303         sq->hwq.max_elements = sq->max_wqe;
 304         rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0,
 305                                        &sq->hwq.max_elements,
 306                                        BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,
 307                                        PAGE_SIZE, HWQ_TYPE_QUEUE);
 308         if (rc)
 309                 goto exit;
 310
 311         sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
 312         if (!sq->swq) {
 313                 rc = -ENOMEM;
 314                 goto fail_sq;
 315         }
 316         pbl = &sq->hwq.pbl[PBL_LVL_0];
 317         req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
 318         req.sq_pg_size_sq_lvl =
 319                 ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK)
 320                                 <<  CMDQ_CREATE_QP1_SQ_LVL_SFT) |
 321                 (pbl->pg_size == ROCE_PG_SIZE_4K ?
 322                                 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K :
 323                  pbl->pg_size == ROCE_PG_SIZE_8K ?
 324                                 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K :
 325                  pbl->pg_size == ROCE_PG_SIZE_64K ?
 326                                 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K :
 327                  pbl->pg_size == ROCE_PG_SIZE_2M ?
 328                                 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M :
 329                  pbl->pg_size == ROCE_PG_SIZE_8M ?
 330                                 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M :
 331                  pbl->pg_size == ROCE_PG_SIZE_1G ?
 332                                 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G :
 333                  CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K);
 334
 335         if (qp->scq)
 336                 req.scq_cid = cpu_to_le32(qp->scq->id);
 337
 338         qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
 339
 340         /* RQ */
 341         if (rq->max_wqe) {
 342                 rq->hwq.max_elements = qp->rq.max_wqe;
 343                 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0,
 344                                                &rq->hwq.max_elements,
 345                                                BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
 346                                                PAGE_SIZE, HWQ_TYPE_QUEUE);
 347                 if (rc)
 348                         goto fail_sq;
 349
 350                 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
 351                                   GFP_KERNEL);
 352                 if (!rq->swq) {
 353                         rc = -ENOMEM;
 354                         goto fail_rq;
 355                 }
 356                 pbl = &rq->hwq.pbl[PBL_LVL_0];
 357                 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
 358                 req.rq_pg_size_rq_lvl =
 359                         ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
 360                          CMDQ_CREATE_QP1_RQ_LVL_SFT) |
 361                                 (pbl->pg_size == ROCE_PG_SIZE_4K ?
 362                                         CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K :
 363                                  pbl->pg_size == ROCE_PG_SIZE_8K ?
 364                                         CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K :
 365                                  pbl->pg_size == ROCE_PG_SIZE_64K ?
 366                                         CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K :
 367                                  pbl->pg_size == ROCE_PG_SIZE_2M ?
 368                                         CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M :
 369                                  pbl->pg_size == ROCE_PG_SIZE_8M ?
 370                                         CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M :
 371                                  pbl->pg_size == ROCE_PG_SIZE_1G ?
 372                                         CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G :
 373                                  CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K);
 374                 if (qp->rcq)
 375                         req.rcq_cid = cpu_to_le32(qp->rcq->id);
 376         }
 377
 378         /* Header buffer - allow hdr_buf pass in */
 379         rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
 380         if (rc) {
 381                 rc = -ENOMEM;
 382                 goto fail;
 383         }
 384         req.qp_flags = cpu_to_le32(qp_flags);
 385         req.sq_size = cpu_to_le32(sq->hwq.max_elements);
 386         req.rq_size = cpu_to_le32(rq->hwq.max_elements);
 387
 388         req.sq_fwo_sq_sge =
 389                 cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
 390                             CMDQ_CREATE_QP1_SQ_SGE_SFT);
 391         req.rq_fwo_rq_sge =
 392                 cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
 393                             CMDQ_CREATE_QP1_RQ_SGE_SFT);
 394
 395         req.pd_id = cpu_to_le32(qp->pd->id);
 396
 397         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
 398                                           (void *)&resp, NULL, 0);
 399         if (rc)
 400                 goto fail;
 401
 402         qp->id = le32_to_cpu(resp.xid);
 403         qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
 404         sq->flush_in_progress = false;
 405         rq->flush_in_progress = false;
 406
 407         return 0;
 408
 409 fail:
 410         bnxt_qplib_free_qp_hdr_buf(res, qp);
 411 fail_rq:
 412         bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
 413         kfree(rq->swq);
 414 fail_sq:
 415         bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
 416         kfree(sq->swq);
 417 exit:
 418         return rc;
 419 }
 420
 421 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 422 {
 423         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 424         struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
 425         struct cmdq_create_qp req;
 426         struct creq_create_qp_resp resp;
 427         struct bnxt_qplib_pbl *pbl;
 428         struct sq_psn_search **psn_search_ptr;
 429         unsigned long int psn_search, poff = 0;
 430         struct bnxt_qplib_q *sq = &qp->sq;
 431         struct bnxt_qplib_q *rq = &qp->rq;
 432         struct bnxt_qplib_hwq *xrrq;
 433         int i, rc, req_size, psn_sz;
 434         u16 cmd_flags = 0, max_ssge;
 435         u32 sw_prod, qp_flags = 0;
 436
 437         RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
 438
 439         /* General */
 440         req.type = qp->type;
 441         req.dpi = cpu_to_le32(qp->dpi->dpi);
 442         req.qp_handle = cpu_to_le64(qp->qp_handle);
 443
 444         /* SQ */
 445         psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ?
 446                  sizeof(struct sq_psn_search) : 0;
 447         sq->hwq.max_elements = sq->max_wqe;
 448         rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist,
 449                                        sq->nmap, &sq->hwq.max_elements,
 450                                        BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,
 451                                        psn_sz,
 452                                        PAGE_SIZE, HWQ_TYPE_QUEUE);
 453         if (rc)
 454                 goto exit;
 455
 456         sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
 457         if (!sq->swq) {
 458                 rc = -ENOMEM;
 459                 goto fail_sq;
 460         }
 461         hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
 462         if (psn_sz) {
 463                 psn_search_ptr = (struct sq_psn_search **)
 464                                   &hw_sq_send_ptr[get_sqe_pg
 465                                         (sq->hwq.max_elements)];
 466                 psn_search = (unsigned long int)
 467                               &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)]
 468                               [get_sqe_idx(sq->hwq.max_elements)];
 469                 if (psn_search & ~PAGE_MASK) {
 470                         /* If the psn_search does not start on a page boundary,
 471                          * then calculate the offset
 472                          */
 473                         poff = (psn_search & ~PAGE_MASK) /
 474                                 BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;
 475                 }
 476                 for (i = 0; i < sq->hwq.max_elements; i++)
 477                         sq->swq[i].psn_search =
 478                                 &psn_search_ptr[get_psne_pg(i + poff)]
 479                                                [get_psne_idx(i + poff)];
 480         }
 481         pbl = &sq->hwq.pbl[PBL_LVL_0];
 482         req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
 483         req.sq_pg_size_sq_lvl =
 484                 ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK)
 485                                  <<  CMDQ_CREATE_QP_SQ_LVL_SFT) |
 486                 (pbl->pg_size == ROCE_PG_SIZE_4K ?
 487                                 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K :
 488                  pbl->pg_size == ROCE_PG_SIZE_8K ?
 489                                 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K :
 490                  pbl->pg_size == ROCE_PG_SIZE_64K ?
 491                                 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K :
 492                  pbl->pg_size == ROCE_PG_SIZE_2M ?
 493                                 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M :
 494                  pbl->pg_size == ROCE_PG_SIZE_8M ?
 495                                 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M :
 496                  pbl->pg_size == ROCE_PG_SIZE_1G ?
 497                                 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G :
 498                  CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K);
 499
 500         /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */
 501         hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
 502         for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) {
 503                 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
 504                                                 [get_sqe_idx(sw_prod)];
 505                 hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID;
 506         }
 507
 508         if (qp->scq)
 509                 req.scq_cid = cpu_to_le32(qp->scq->id);
 510
 511         qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
 512         qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
 513         if (qp->sig_type)
 514                 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
 515
 516         /* RQ */
 517         if (rq->max_wqe) {
 518                 rq->hwq.max_elements = rq->max_wqe;
 519                 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist,
 520                                                rq->nmap, &rq->hwq.max_elements,
 521                                                BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
 522                                                PAGE_SIZE, HWQ_TYPE_QUEUE);
 523                 if (rc)
 524                         goto fail_sq;
 525
 526                 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
 527                                   GFP_KERNEL);
 528                 if (!rq->swq) {
 529                         rc = -ENOMEM;
 530                         goto fail_rq;
 531                 }
 532                 pbl = &rq->hwq.pbl[PBL_LVL_0];
 533                 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
 534                 req.rq_pg_size_rq_lvl =
 535                         ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
 536                          CMDQ_CREATE_QP_RQ_LVL_SFT) |
 537                                 (pbl->pg_size == ROCE_PG_SIZE_4K ?
 538                                         CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K :
 539                                  pbl->pg_size == ROCE_PG_SIZE_8K ?
 540                                         CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K :
 541                                  pbl->pg_size == ROCE_PG_SIZE_64K ?
 542                                         CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K :
 543                                  pbl->pg_size == ROCE_PG_SIZE_2M ?
 544                                         CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M :
 545                                  pbl->pg_size == ROCE_PG_SIZE_8M ?
 546                                         CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M :
 547                                  pbl->pg_size == ROCE_PG_SIZE_1G ?
 548                                         CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :
 549                                  CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);
 550         }
 551
 552         if (qp->rcq)
 553                 req.rcq_cid = cpu_to_le32(qp->rcq->id);
 554         req.qp_flags = cpu_to_le32(qp_flags);
 555         req.sq_size = cpu_to_le32(sq->hwq.max_elements);
 556         req.rq_size = cpu_to_le32(rq->hwq.max_elements);
 557         qp->sq_hdr_buf = NULL;
 558         qp->rq_hdr_buf = NULL;
 559
 560         rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
 561         if (rc)
 562                 goto fail_rq;
 563
 564         /* CTRL-22434: Irrespective of the requested SGE count on the SQ
 565          * always create the QP with max send sges possible if the requested
 566          * inline size is greater than 0.
 567          */
 568         max_ssge = qp->max_inline_data ? 6 : sq->max_sge;
 569         req.sq_fwo_sq_sge = cpu_to_le16(
 570                                 ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
 571                                  << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
 572         req.rq_fwo_rq_sge = cpu_to_le16(
 573                                 ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK)
 574                                  << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
 575         /* ORRQ and IRRQ */
 576         if (psn_sz) {
 577                 xrrq = &qp->orrq;
 578                 xrrq->max_elements =
 579                         ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
 580                 req_size = xrrq->max_elements *
 581                            BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
 582                 req_size &= ~(PAGE_SIZE - 1);
 583                 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
 584                                                &xrrq->max_elements,
 585                                                BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,
 586                                                0, req_size, HWQ_TYPE_CTX);
 587                 if (rc)
 588                         goto fail_buf_free;
 589                 pbl = &xrrq->pbl[PBL_LVL_0];
 590                 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
 591
 592                 xrrq = &qp->irrq;
 593                 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
 594                                                 qp->max_dest_rd_atomic);
 595                 req_size = xrrq->max_elements *
 596                            BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
 597                 req_size &= ~(PAGE_SIZE - 1);
 598
 599                 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
 600                                                &xrrq->max_elements,
 601                                                BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,
 602                                                0, req_size, HWQ_TYPE_CTX);
 603                 if (rc)
 604                         goto fail_orrq;
 605
 606                 pbl = &xrrq->pbl[PBL_LVL_0];
 607                 req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
 608         }
 609         req.pd_id = cpu_to_le32(qp->pd->id);
 610
 611         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
 612                                           (void *)&resp, NULL, 0);
 613         if (rc)
 614                 goto fail;
 615
 616         qp->id = le32_to_cpu(resp.xid);
 617         qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
 618         sq->flush_in_progress = false;
 619         rq->flush_in_progress = false;
 620
 621         return 0;
 622
 623 fail:
 624         if (qp->irrq.max_elements)
 625                 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
 626 fail_orrq:
 627         if (qp->orrq.max_elements)
 628                 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
 629 fail_buf_free:
 630         bnxt_qplib_free_qp_hdr_buf(res, qp);
 631 fail_rq:
 632         bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
 633         kfree(rq->swq);
 634 fail_sq:
 635         bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
 636         kfree(sq->swq);
 637 exit:
 638         return rc;
 639 }
 640
 641 static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp)
 642 {
 643         switch (qp->state) {
 644         case CMDQ_MODIFY_QP_NEW_STATE_RTR:
 645                 /* INIT->RTR, configure the path_mtu to the default
 646                  * 2048 if not being requested
 647                  */
 648                 if (!(qp->modify_flags &
 649                     CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
 650                         qp->modify_flags |=
 651                                 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
 652                         qp->path_mtu =
 653                                 CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
 654                 }
 655                 qp->modify_flags &=
 656                         ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
 657                 /* Bono FW require the max_dest_rd_atomic to be >= 1 */
 658                 if (qp->max_dest_rd_atomic < 1)
 659                         qp->max_dest_rd_atomic = 1;
 660                 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
 661                 /* Bono FW 20.6.5 requires SGID_INDEX configuration */
 662                 if (!(qp->modify_flags &
 663                     CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
 664                         qp->modify_flags |=
 665                                 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
 666                         qp->ah.sgid_index = 0;
 667                 }
 668                 break;
 669         default:
 670                 break;
 671         }
 672 }
 673
 674 static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp)
 675 {
 676         switch (qp->state) {
 677         case CMDQ_MODIFY_QP_NEW_STATE_RTS:
 678                 /* Bono FW requires the max_rd_atomic to be >= 1 */
 679                 if (qp->max_rd_atomic < 1)
 680                         qp->max_rd_atomic = 1;
 681                 /* Bono FW does not allow PKEY_INDEX,
 682                  * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT,
 683                  * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN,
 684                  * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID
 685                  * modification
 686                  */
 687                 qp->modify_flags &=
 688                         ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
 689                           CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
 690                           CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
 691                           CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
 692                           CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
 693                           CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
 694                           CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
 695                           CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
 696                           CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
 697                           CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
 698                           CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
 699                           CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
 700                 break;
 701         default:
 702                 break;
 703         }
 704 }
 705
 706 static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
 707 {
 708         switch (qp->cur_qp_state) {
 709         case CMDQ_MODIFY_QP_NEW_STATE_RESET:
 710                 break;
 711         case CMDQ_MODIFY_QP_NEW_STATE_INIT:
 712                 __modify_flags_from_init_state(qp);
 713                 break;
 714         case CMDQ_MODIFY_QP_NEW_STATE_RTR:
 715                 __modify_flags_from_rtr_state(qp);
 716                 break;
 717         case CMDQ_MODIFY_QP_NEW_STATE_RTS:
 718                 break;
 719         case CMDQ_MODIFY_QP_NEW_STATE_SQD:
 720                 break;
 721         case CMDQ_MODIFY_QP_NEW_STATE_SQE:
 722                 break;
 723         case CMDQ_MODIFY_QP_NEW_STATE_ERR:
 724                 break;
 725         default:
 726                 break;
 727         }
 728 }
 729
 730 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 731 {
 732         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 733         struct cmdq_modify_qp req;
 734         struct creq_modify_qp_resp resp;
 735         u16 cmd_flags = 0, pkey;
 736         u32 temp32[4];
 737         u32 bmask;
 738         int rc;
 739
 740         RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags);
 741
 742         /* Filter out the qp_attr_mask based on the state->new transition */
 743         __filter_modify_flags(qp);
 744         bmask = qp->modify_flags;
 745         req.modify_mask = cpu_to_le32(qp->modify_flags);
 746         req.qp_cid = cpu_to_le32(qp->id);
 747         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
 748                 req.network_type_en_sqd_async_notify_new_state =
 749                                 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
 750                                 (qp->en_sqd_async_notify ?
 751                                         CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
 752         }
 753         req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;
 754
 755         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)
 756                 req.access = qp->access;
 757
 758         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) {
 759                 if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl,
 760                                          qp->pkey_index, &pkey))
 761                         req.pkey = cpu_to_le16(pkey);
 762         }
 763         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)
 764                 req.qkey = cpu_to_le32(qp->qkey);
 765
 766         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
 767                 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));
 768                 req.dgid[0] = cpu_to_le32(temp32[0]);
 769                 req.dgid[1] = cpu_to_le32(temp32[1]);
 770                 req.dgid[2] = cpu_to_le32(temp32[2]);
 771                 req.dgid[3] = cpu_to_le32(temp32[3]);
 772         }
 773         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL)
 774                 req.flow_label = cpu_to_le32(qp->ah.flow_label);
 775
 776         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)
 777                 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id
 778                                              [qp->ah.sgid_index]);
 779
 780         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT)
 781                 req.hop_limit = qp->ah.hop_limit;
 782
 783         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS)
 784                 req.traffic_class = qp->ah.traffic_class;
 785
 786         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC)
 787                 memcpy(req.dest_mac, qp->ah.dmac, 6);
 788
 789         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)
 790                 req.path_mtu = qp->path_mtu;
 791
 792         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT)
 793                 req.timeout = qp->timeout;
 794
 795         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT)
 796                 req.retry_cnt = qp->retry_cnt;
 797
 798         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY)
 799                 req.rnr_retry = qp->rnr_retry;
 800
 801         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER)
 802                 req.min_rnr_timer = qp->min_rnr_timer;
 803
 804         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN)
 805                 req.rq_psn = cpu_to_le32(qp->rq.psn);
 806
 807         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN)
 808                 req.sq_psn = cpu_to_le32(qp->sq.psn);
 809
 810         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC)
 811                 req.max_rd_atomic =
 812                         ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
 813
 814         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC)
 815                 req.max_dest_rd_atomic =
 816                         IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);
 817
 818         req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);
 819         req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);
 820         req.sq_sge = cpu_to_le16(qp->sq.max_sge);
 821         req.rq_sge = cpu_to_le16(qp->rq.max_sge);
 822         req.max_inline_data = cpu_to_le32(qp->max_inline_data);
 823         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
 824                 req.dest_qp_id = cpu_to_le32(qp->dest_qpn);
 825
 826         req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
 827
 828         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
 829                                           (void *)&resp, NULL, 0);
 830         if (rc)
 831                 return rc;
 832         qp->cur_qp_state = qp->state;
 833         return 0;
 834 }
 835
 836 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 837 {
 838         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 839         struct cmdq_query_qp req;
 840         struct creq_query_qp_resp resp;
 841         struct bnxt_qplib_rcfw_sbuf *sbuf;
 842         struct creq_query_qp_resp_sb *sb;
 843         u16 cmd_flags = 0;
 844         u32 temp32[4];
 845         int i, rc = 0;
 846
 847         RCFW_CMD_PREP(req, QUERY_QP, cmd_flags);
 848
 849         sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
 850         if (!sbuf)
 851                 return -ENOMEM;
 852         sb = sbuf->sb;
 853
 854         req.qp_cid = cpu_to_le32(qp->id);
 855         req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
 856         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
 857                                           (void *)sbuf, 0);
 858         if (rc)
 859                 goto bail;
 860         /* Extract the context from the side buffer */
 861         qp->state = sb->en_sqd_async_notify_state &
 862                         CREQ_QUERY_QP_RESP_SB_STATE_MASK;
 863         qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &
 864                                   CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?
 865                                   true : false;
 866         qp->access = sb->access;
 867         qp->pkey_index = le16_to_cpu(sb->pkey);
 868         qp->qkey = le32_to_cpu(sb->qkey);
 869
 870         temp32[0] = le32_to_cpu(sb->dgid[0]);
 871         temp32[1] = le32_to_cpu(sb->dgid[1]);
 872         temp32[2] = le32_to_cpu(sb->dgid[2]);
 873         temp32[3] = le32_to_cpu(sb->dgid[3]);
 874         memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));
 875
 876         qp->ah.flow_label = le32_to_cpu(sb->flow_label);
 877
 878         qp->ah.sgid_index = 0;
 879         for (i = 0; i < res->sgid_tbl.max; i++) {
 880                 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {
 881                         qp->ah.sgid_index = i;
 882                         break;
 883                 }
 884         }
 885         if (i == res->sgid_tbl.max)
 886                 dev_warn(&res->pdev->dev, "QPLIB: SGID not found??");
 887
 888         qp->ah.hop_limit = sb->hop_limit;
 889         qp->ah.traffic_class = sb->traffic_class;
 890         memcpy(qp->ah.dmac, sb->dest_mac, 6);
 891         qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
 892                                 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >>
 893                                 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT;
 894         qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
 895                                     CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >>
 896                                     CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT;
 897         qp->timeout = sb->timeout;
 898         qp->retry_cnt = sb->retry_cnt;
 899         qp->rnr_retry = sb->rnr_retry;
 900         qp->min_rnr_timer = sb->min_rnr_timer;
 901         qp->rq.psn = le32_to_cpu(sb->rq_psn);
 902         qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);
 903         qp->sq.psn = le32_to_cpu(sb->sq_psn);
 904         qp->max_dest_rd_atomic =
 905                         IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);
 906         qp->sq.max_wqe = qp->sq.hwq.max_elements;
 907         qp->rq.max_wqe = qp->rq.hwq.max_elements;
 908         qp->sq.max_sge = le16_to_cpu(sb->sq_sge);
 909         qp->rq.max_sge = le16_to_cpu(sb->rq_sge);
 910         qp->max_inline_data = le32_to_cpu(sb->max_inline_data);
 911         qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
 912         memcpy(qp->smac, sb->src_mac, 6);
 913         qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
 914 bail:
 915         bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
 916         return rc;
 917 }
 918
 919 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
 920 {
 921         struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
 922         struct cq_base *hw_cqe, **hw_cqe_ptr;
 923         int i;
 924
 925         for (i = 0; i < cq_hwq->max_elements; i++) {
 926                 hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr;
 927                 hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)];
 928                 if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
 929                         continue;
 930                 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
 931                 case CQ_BASE_CQE_TYPE_REQ:
 932                 case CQ_BASE_CQE_TYPE_TERMINAL:
 933                 {
 934                         struct cq_req *cqe = (struct cq_req *)hw_cqe;
 935
 936                         if (qp == le64_to_cpu(cqe->qp_handle))
 937                                 cqe->qp_handle = 0;
 938                         break;
 939                 }
 940                 case CQ_BASE_CQE_TYPE_RES_RC:
 941                 case CQ_BASE_CQE_TYPE_RES_UD:
 942                 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
 943                 {
 944                         struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
 945
 946                         if (qp == le64_to_cpu(cqe->qp_handle))
 947                                 cqe->qp_handle = 0;
 948                         break;
 949                 }
 950                 default:
 951                         break;
 952                 }
 953         }
 954 }
 955
 956 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
 957                           struct bnxt_qplib_qp *qp)
 958 {
 959         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 960         struct cmdq_destroy_qp req;
 961         struct creq_destroy_qp_resp resp;
 962         unsigned long flags;
 963         u16 cmd_flags = 0;
 964         int rc;
 965
 966         RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
 967
 968         req.qp_cid = cpu_to_le32(qp->id);
 969         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
 970                                           (void *)&resp, NULL, 0);
 971         if (rc)
 972                 return rc;
 973
 974         /* Must walk the associated CQs to nullified the QP ptr */
 975         spin_lock_irqsave(&qp->scq->hwq.lock, flags);
 976
 977         __clean_cq(qp->scq, (u64)(unsigned long)qp);
 978
 979         if (qp->rcq && qp->rcq != qp->scq) {
 980                 spin_lock(&qp->rcq->hwq.lock);
 981                 __clean_cq(qp->rcq, (u64)(unsigned long)qp);
 982                 spin_unlock(&qp->rcq->hwq.lock);
 983         }
 984
 985         spin_unlock_irqrestore(&qp->scq->hwq.lock, flags);
 986
 987         bnxt_qplib_free_qp_hdr_buf(res, qp);
 988         bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
 989         kfree(qp->sq.swq);
 990
 991         bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq);
 992         kfree(qp->rq.swq);
 993
 994         if (qp->irrq.max_elements)
 995                 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
 996         if (qp->orrq.max_elements)
 997                 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
 998
 999         return 0;
1000 }
1001
1002 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
1003                                 struct bnxt_qplib_sge *sge)
1004 {
1005         struct bnxt_qplib_q *sq = &qp->sq;
1006         u32 sw_prod;
1007
1008         memset(sge, 0, sizeof(*sge));
1009
1010         if (qp->sq_hdr_buf) {
1011                 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1012                 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
1013                                          sw_prod * qp->sq_hdr_buf_size);
1014                 sge->lkey = 0xFFFFFFFF;
1015                 sge->size = qp->sq_hdr_buf_size;
1016                 return qp->sq_hdr_buf + sw_prod * sge->size;
1017         }
1018         return NULL;
1019 }
1020
1021 u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)
1022 {
1023         struct bnxt_qplib_q *rq = &qp->rq;
1024
1025         return HWQ_CMP(rq->hwq.prod, &rq->hwq);
1026 }
1027
1028 dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
1029 {
1030         return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size);
1031 }
1032
1033 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
1034                                 struct bnxt_qplib_sge *sge)
1035 {
1036         struct bnxt_qplib_q *rq = &qp->rq;
1037         u32 sw_prod;
1038
1039         memset(sge, 0, sizeof(*sge));
1040
1041         if (qp->rq_hdr_buf) {
1042                 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1043                 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
1044                                          sw_prod * qp->rq_hdr_buf_size);
1045                 sge->lkey = 0xFFFFFFFF;
1046                 sge->size = qp->rq_hdr_buf_size;
1047                 return qp->rq_hdr_buf + sw_prod * sge->size;
1048         }
1049         return NULL;
1050 }
1051
1052 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1053 {
1054         struct bnxt_qplib_q *sq = &qp->sq;
1055         struct dbr_dbr db_msg = { 0 };
1056         u32 sw_prod;
1057
1058         sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1059
1060         db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1061                                    DBR_DBR_INDEX_MASK);
1062         db_msg.type_xid =
1063                 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1064                             DBR_DBR_TYPE_SQ);
1065         /* Flush all the WQE writes to HW */
1066         wmb();
1067         __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1068 }
1069
1070 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
1071                          struct bnxt_qplib_swqe *wqe)
1072 {
1073         struct bnxt_qplib_q *sq = &qp->sq;
1074         struct bnxt_qplib_swq *swq;
1075         struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
1076         struct sq_sge *hw_sge;
1077         u32 sw_prod;
1078         u8 wqe_size16;
1079         int i, rc = 0, data_len = 0, pkt_num = 0;
1080         __le32 temp32;
1081
1082         if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
1083                 rc = -EINVAL;
1084                 goto done;
1085         }
1086
1087         if (bnxt_qplib_queue_full(sq)) {
1088                 dev_err(&sq->hwq.pdev->dev,
1089                         "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x",
1090                         sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements,
1091                         sq->q_full_delta);
1092                 rc = -ENOMEM;
1093                 goto done;
1094         }
1095         sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1096         swq = &sq->swq[sw_prod];
1097         swq->wr_id = wqe->wr_id;
1098         swq->type = wqe->type;
1099         swq->flags = wqe->flags;
1100         if (qp->sig_type)
1101                 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1102         swq->start_psn = sq->psn & BTH_PSN_MASK;
1103
1104         hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
1105         hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
1106                                         [get_sqe_idx(sw_prod)];
1107
1108         memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
1109
1110         if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
1111                 /* Copy the inline data */
1112                 if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1113                         dev_warn(&sq->hwq.pdev->dev,
1114                                  "QPLIB: Inline data length > 96 detected");
1115                         data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;
1116                 } else {
1117                         data_len = wqe->inline_len;
1118                 }
1119                 memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len);
1120                 wqe_size16 = (data_len + 15) >> 4;
1121         } else {
1122                 for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data;
1123                      i < wqe->num_sge; i++, hw_sge++) {
1124                         hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1125                         hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1126                         hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1127                         data_len += wqe->sg_list[i].size;
1128                 }
1129                 /* Each SGE entry = 1 WQE size16 */
1130                 wqe_size16 = wqe->num_sge;
1131                 /* HW requires wqe size has room for atleast one SGE even if
1132                  * none was supplied by ULP
1133                  */
1134                 if (!wqe->num_sge)
1135                         wqe_size16++;
1136         }
1137
1138         /* Specifics */
1139         switch (wqe->type) {
1140         case BNXT_QPLIB_SWQE_TYPE_SEND:
1141                 if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
1142                         /* Assemble info for Raw Ethertype QPs */
1143                         struct sq_send_raweth_qp1 *sqe =
1144                                 (struct sq_send_raweth_qp1 *)hw_sq_send_hdr;
1145
1146                         sqe->wqe_type = wqe->type;
1147                         sqe->flags = wqe->flags;
1148                         sqe->wqe_size = wqe_size16 +
1149                                 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1150                         sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
1151                         sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
1152                         sqe->length = cpu_to_le32(data_len);
1153                         sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
1154                                 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
1155                                 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
1156
1157                         break;
1158                 }
1159                 /* else, just fall thru */
1160         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
1161         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
1162         {
1163                 struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr;
1164
1165                 sqe->wqe_type = wqe->type;
1166                 sqe->flags = wqe->flags;
1167                 sqe->wqe_size = wqe_size16 +
1168                                 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1169                 sqe->inv_key_or_imm_data = cpu_to_le32(
1170                                                 wqe->send.inv_key);
1171                 if (qp->type == CMDQ_CREATE_QP_TYPE_UD) {
1172                         sqe->q_key = cpu_to_le32(wqe->send.q_key);
1173                         sqe->dst_qp = cpu_to_le32(
1174                                         wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
1175                         sqe->length = cpu_to_le32(data_len);
1176                         sqe->avid = cpu_to_le32(wqe->send.avid &
1177                                                 SQ_SEND_AVID_MASK);
1178                         sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
1179                 } else {
1180                         sqe->length = cpu_to_le32(data_len);
1181                         sqe->dst_qp = 0;
1182                         sqe->avid = 0;
1183                         if (qp->mtu)
1184                                 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1185                         if (!pkt_num)
1186                                 pkt_num = 1;
1187                         sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1188                 }
1189                 break;
1190         }
1191         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
1192         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
1193         case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
1194         {
1195                 struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr;
1196
1197                 sqe->wqe_type = wqe->type;
1198                 sqe->flags = wqe->flags;
1199                 sqe->wqe_size = wqe_size16 +
1200                                 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1201                 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
1202                 sqe->length = cpu_to_le32((u32)data_len);
1203                 sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
1204                 sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
1205                 if (qp->mtu)
1206                         pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1207                 if (!pkt_num)
1208                         pkt_num = 1;
1209                 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1210                 break;
1211         }
1212         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
1213         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
1214         {
1215                 struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr;
1216
1217                 sqe->wqe_type = wqe->type;
1218                 sqe->flags = wqe->flags;
1219                 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
1220                 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
1221                 sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
1222                 sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
1223                 if (qp->mtu)
1224                         pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1225                 if (!pkt_num)
1226                         pkt_num = 1;
1227                 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1228                 break;
1229         }
1230         case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
1231         {
1232                 struct sq_localinvalidate *sqe =
1233                                 (struct sq_localinvalidate *)hw_sq_send_hdr;
1234
1235                 sqe->wqe_type = wqe->type;
1236                 sqe->flags = wqe->flags;
1237                 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);
1238
1239                 break;
1240         }
1241         case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
1242         {
1243                 struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr;
1244
1245                 sqe->wqe_type = wqe->type;
1246                 sqe->flags = wqe->flags;
1247                 sqe->access_cntl = wqe->frmr.access_cntl |
1248                                    SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
1249                 sqe->zero_based_page_size_log =
1250                         (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<
1251                         SQ_FR_PMR_PAGE_SIZE_LOG_SFT |
1252                         (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0);
1253                 sqe->l_key = cpu_to_le32(wqe->frmr.l_key);
1254                 temp32 = cpu_to_le32(wqe->frmr.length);
1255                 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));
1256                 sqe->numlevels_pbl_page_size_log =
1257                         ((wqe->frmr.pbl_pg_sz_log <<
1258                                         SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) &
1259                                         SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) |
1260                         ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &
1261                                         SQ_FR_PMR_NUMLEVELS_MASK);
1262
1263                 for (i = 0; i < wqe->frmr.page_list_len; i++)
1264                         wqe->frmr.pbl_ptr[i] = cpu_to_le64(
1265                                                 wqe->frmr.page_list[i] |
1266                                                 PTU_PTE_VALID);
1267                 sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
1268                 sqe->va = cpu_to_le64(wqe->frmr.va);
1269
1270                 break;
1271         }
1272         case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
1273         {
1274                 struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr;
1275
1276                 sqe->wqe_type = wqe->type;
1277                 sqe->flags = wqe->flags;
1278                 sqe->access_cntl = wqe->bind.access_cntl;
1279                 sqe->mw_type_zero_based = wqe->bind.mw_type |
1280                         (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
1281                 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
1282                 sqe->l_key = cpu_to_le32(wqe->bind.r_key);
1283                 sqe->va = cpu_to_le64(wqe->bind.va);
1284                 temp32 = cpu_to_le32(wqe->bind.length);
1285                 memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length));
1286                 break;
1287         }
1288         default:
1289                 /* Bad wqe, return error */
1290                 rc = -EINVAL;
1291                 goto done;
1292         }
1293         swq->next_psn = sq->psn & BTH_PSN_MASK;
1294         if (swq->psn_search) {
1295                 swq->psn_search->opcode_start_psn = cpu_to_le32(
1296                         ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
1297                          SQ_PSN_SEARCH_START_PSN_MASK) |
1298                         ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
1299                          SQ_PSN_SEARCH_OPCODE_MASK));
1300                 swq->psn_search->flags_next_psn = cpu_to_le32(
1301                         ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
1302                          SQ_PSN_SEARCH_NEXT_PSN_MASK));
1303         }
1304
1305         sq->hwq.prod++;
1306
1307         qp->wqe_cnt++;
1308
1309 done:
1310         return rc;
1311 }
1312
1313 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
1314 {
1315         struct bnxt_qplib_q *rq = &qp->rq;
1316         struct dbr_dbr db_msg = { 0 };
1317         u32 sw_prod;
1318
1319         sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1320         db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1321                                    DBR_DBR_INDEX_MASK);
1322         db_msg.type_xid =
1323                 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1324                             DBR_DBR_TYPE_RQ);
1325
1326         /* Flush the writes to HW Rx WQE before the ringing Rx DB */
1327         wmb();
1328         __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1329 }
1330
1331 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
1332                          struct bnxt_qplib_swqe *wqe)
1333 {
1334         struct bnxt_qplib_q *rq = &qp->rq;
1335         struct rq_wqe *rqe, **rqe_ptr;
1336         struct sq_sge *hw_sge;
1337         u32 sw_prod;
1338         int i, rc = 0;
1339
1340         if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1341                 dev_err(&rq->hwq.pdev->dev,
1342                         "QPLIB: FP: QP (0x%x) is in the 0x%x state",
1343                         qp->id, qp->state);
1344                 rc = -EINVAL;
1345                 goto done;
1346         }
1347         if (bnxt_qplib_queue_full(rq)) {
1348                 dev_err(&rq->hwq.pdev->dev,
1349                         "QPLIB: FP: QP (0x%x) RQ is full!", qp->id);
1350                 rc = -EINVAL;
1351                 goto done;
1352         }
1353         sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1354         rq->swq[sw_prod].wr_id = wqe->wr_id;
1355
1356         rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr;
1357         rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
1358
1359         memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
1360
1361         /* Calculate wqe_size16 and data_len */
1362         for (i = 0, hw_sge = (struct sq_sge *)rqe->data;
1363              i < wqe->num_sge; i++, hw_sge++) {
1364                 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1365                 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1366                 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1367         }
1368         rqe->wqe_type = wqe->type;
1369         rqe->flags = wqe->flags;
1370         rqe->wqe_size = wqe->num_sge +
1371                         ((offsetof(typeof(*rqe), data) + 15) >> 4);
1372         /* HW requires wqe size has room for atleast one SGE even if none
1373          * was supplied by ULP
1374          */
1375         if (!wqe->num_sge)
1376                 rqe->wqe_size++;
1377
1378         /* Supply the rqe->wr_id index to the wr_id_tbl for now */
1379         rqe->wr_id[0] = cpu_to_le32(sw_prod);
1380
1381         rq->hwq.prod++;
1382 done:
1383         return rc;
1384 }
1385
1386 /* CQ */
1387
1388 /* Spinlock must be held */
1389 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq)
1390 {
1391         struct dbr_dbr db_msg = { 0 };
1392
1393         db_msg.type_xid =
1394                 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1395                             DBR_DBR_TYPE_CQ_ARMENA);
1396         /* Flush memory writes before enabling the CQ */
1397         wmb();
1398         __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64));
1399 }
1400
1401 static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
1402 {
1403         struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1404         struct dbr_dbr db_msg = { 0 };
1405         u32 sw_cons;
1406
1407         /* Ring DB */
1408         sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq);
1409         db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) &
1410                                     DBR_DBR_INDEX_MASK);
1411         db_msg.type_xid =
1412                 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1413                             arm_type);
1414         /* flush memory writes before arming the CQ */
1415         wmb();
1416         __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1417 }
1418
1419 int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1420 {
1421         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1422         struct cmdq_create_cq req;
1423         struct creq_create_cq_resp resp;
1424         struct bnxt_qplib_pbl *pbl;
1425         u16 cmd_flags = 0;
1426         int rc;
1427
1428         cq->hwq.max_elements = cq->max_wqe;
1429         rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead,
1430                                        cq->nmap, &cq->hwq.max_elements,
1431                                        BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,
1432                                        PAGE_SIZE, HWQ_TYPE_QUEUE);
1433         if (rc)
1434                 goto exit;
1435
1436         RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);
1437
1438         if (!cq->dpi) {
1439                 dev_err(&rcfw->pdev->dev,
1440                         "QPLIB: FP: CREATE_CQ failed due to NULL DPI");
1441                 return -EINVAL;
1442         }
1443         req.dpi = cpu_to_le32(cq->dpi->dpi);
1444         req.cq_handle = cpu_to_le64(cq->cq_handle);
1445
1446         req.cq_size = cpu_to_le32(cq->hwq.max_elements);
1447         pbl = &cq->hwq.pbl[PBL_LVL_0];
1448         req.pg_size_lvl = cpu_to_le32(
1449             ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
1450                                                 CMDQ_CREATE_CQ_LVL_SFT) |
1451             (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K :
1452              pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K :
1453              pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K :
1454              pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M :
1455              pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M :
1456              pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G :
1457              CMDQ_CREATE_CQ_PG_SIZE_PG_4K));
1458
1459         req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
1460
1461         req.cq_fco_cnq_id = cpu_to_le32(
1462                         (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
1463                          CMDQ_CREATE_CQ_CNQ_ID_SFT);
1464
1465         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1466                                           (void *)&resp, NULL, 0);
1467         if (rc)
1468                 goto fail;
1469
1470         cq->id = le32_to_cpu(resp.xid);
1471         cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
1472         cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
1473         init_waitqueue_head(&cq->waitq);
1474
1475         bnxt_qplib_arm_cq_enable(cq);
1476         return 0;
1477
1478 fail:
1479         bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1480 exit:
1481         return rc;
1482 }
1483
1484 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1485 {
1486         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1487         struct cmdq_destroy_cq req;
1488         struct creq_destroy_cq_resp resp;
1489         u16 cmd_flags = 0;
1490         int rc;
1491
1492         RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags);
1493
1494         req.cq_cid = cpu_to_le32(cq->id);
1495         rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1496                                           (void *)&resp, NULL, 0);
1497         if (rc)
1498                 return rc;
1499         bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1500         return 0;
1501 }
1502
1503 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
1504                       struct bnxt_qplib_cqe **pcqe, int *budget)
1505 {
1506         u32 sw_prod, sw_cons;
1507         struct bnxt_qplib_cqe *cqe;
1508         int rc = 0;
1509
1510         /* Now complete all outstanding SQEs with FLUSHED_ERR */
1511         sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1512         cqe = *pcqe;
1513         while (*budget) {
1514                 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1515                 if (sw_cons == sw_prod) {
1516                         sq->flush_in_progress = false;
1517                         break;
1518                 }
1519                 memset(cqe, 0, sizeof(*cqe));
1520                 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
1521                 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1522                 cqe->qp_handle = (u64)(unsigned long)qp;
1523                 cqe->wr_id = sq->swq[sw_cons].wr_id;
1524                 cqe->src_qp = qp->id;
1525                 cqe->type = sq->swq[sw_cons].type;
1526                 cqe++;
1527                 (*budget)--;
1528                 sq->hwq.cons++;
1529         }
1530         *pcqe = cqe;
1531         if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)
1532                 /* Out of budget */
1533                 rc = -EAGAIN;
1534
1535         return rc;
1536 }
1537
1538 static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
1539                       int opcode, struct bnxt_qplib_cqe **pcqe, int *budget)
1540 {
1541         struct bnxt_qplib_cqe *cqe;
1542         u32 sw_prod, sw_cons;
1543         int rc = 0;
1544
1545         /* Flush the rest of the RQ */
1546         sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1547         cqe = *pcqe;
1548         while (*budget) {
1549                 sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);
1550                 if (sw_cons == sw_prod)
1551                         break;
1552                 memset(cqe, 0, sizeof(*cqe));
1553                 cqe->status =
1554                     CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
1555                 cqe->opcode = opcode;
1556                 cqe->qp_handle = (unsigned long)qp;
1557                 cqe->wr_id = rq->swq[sw_cons].wr_id;
1558                 cqe++;
1559                 (*budget)--;
1560                 rq->hwq.cons++;
1561         }
1562         *pcqe = cqe;
1563         if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)
1564                 /* Out of budget */
1565                 rc = -EAGAIN;
1566
1567         return rc;
1568 }
1569
1570 /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive)
1571  *       CQE is track from sw_cq_cons to max_element but valid only if VALID=1
1572  */
1573 static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
1574                      u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons)
1575 {
1576         struct bnxt_qplib_q *sq = &qp->sq;
1577         struct bnxt_qplib_swq *swq;
1578         u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;
1579         struct cq_base *peek_hwcqe, **peek_hw_cqe_ptr;
1580         struct cq_req *peek_req_hwcqe;
1581         struct bnxt_qplib_qp *peek_qp;
1582         struct bnxt_qplib_q *peek_sq;
1583         int i, rc = 0;
1584
1585         /* Normal mode */
1586         /* Check for the psn_search marking before completing */
1587         swq = &sq->swq[sw_sq_cons];
1588         if (swq->psn_search &&
1589             le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
1590                 /* Unmark */
1591                 swq->psn_search->flags_next_psn = cpu_to_le32
1592                         (le32_to_cpu(swq->psn_search->flags_next_psn)
1593                                      & ~0x80000000);
1594                 dev_dbg(&cq->hwq.pdev->dev,
1595                         "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
1596                         cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
1597                 sq->condition = true;
1598                 sq->send_phantom = true;
1599
1600                 /* TODO: Only ARM if the previous SQE is ARMALL */
1601                 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ_ARMALL);
1602
1603                 rc = -EAGAIN;
1604                 goto out;
1605         }
1606         if (sq->condition) {
1607                 /* Peek at the completions */
1608                 peek_raw_cq_cons = cq->hwq.cons;
1609                 peek_sw_cq_cons = cq_cons;
1610                 i = cq->hwq.max_elements;
1611                 while (i--) {
1612                         peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq);
1613                         peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
1614                         peek_hwcqe = &peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)]
1615                                                      [CQE_IDX(peek_sw_cq_cons)];
1616                         /* If the next hwcqe is VALID */
1617                         if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons,
1618                                           cq->hwq.max_elements)) {
1619                                 /* If the next hwcqe is a REQ */
1620                                 if ((peek_hwcqe->cqe_type_toggle &
1621                                     CQ_BASE_CQE_TYPE_MASK) ==
1622                                     CQ_BASE_CQE_TYPE_REQ) {
1623                                         peek_req_hwcqe = (struct cq_req *)
1624                                                          peek_hwcqe;
1625                                         peek_qp = (struct bnxt_qplib_qp *)
1626                                                 ((unsigned long)
1627                                                  le64_to_cpu
1628                                                  (peek_req_hwcqe->qp_handle));
1629                                         peek_sq = &peek_qp->sq;
1630                                         peek_sq_cons_idx = HWQ_CMP(le16_to_cpu(
1631                                                 peek_req_hwcqe->sq_cons_idx) - 1
1632                                                 , &sq->hwq);
1633                                         /* If the hwcqe's sq's wr_id matches */
1634                                         if (peek_sq == sq &&
1635                                             sq->swq[peek_sq_cons_idx].wr_id ==
1636                                             BNXT_QPLIB_FENCE_WRID) {
1637                                                 /*
1638                                                  *  Unbreak only if the phantom
1639                                                  *  comes back
1640                                                  */
1641                                                 dev_dbg(&cq->hwq.pdev->dev,
1642                                                         "FP:Got Phantom CQE");
1643                                                 sq->condition = false;
1644                                                 sq->single = true;
1645                                                 rc = 0;
1646                                                 goto out;
1647                                         }
1648                                 }
1649                                 /* Valid but not the phantom, so keep looping */
1650                         } else {
1651                                 /* Not valid yet, just exit and wait */
1652                                 rc = -EINVAL;
1653                                 goto out;
1654                         }
1655                         peek_sw_cq_cons++;
1656                         peek_raw_cq_cons++;
1657                 }
1658                 dev_err(&cq->hwq.pdev->dev,
1659                         "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x",
1660                         cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
1661                 rc = -EINVAL;
1662         }
1663 out:
1664         return rc;
1665 }
1666
1667 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
1668                                      struct cq_req *hwcqe,
1669                                      struct bnxt_qplib_cqe **pcqe, int *budget,
1670                                      u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
1671 {
1672         struct bnxt_qplib_qp *qp;
1673         struct bnxt_qplib_q *sq;
1674         struct bnxt_qplib_cqe *cqe;
1675         u32 sw_sq_cons, cqe_sq_cons;
1676         struct bnxt_qplib_swq *swq;
1677         int rc = 0;
1678
1679         qp = (struct bnxt_qplib_qp *)((unsigned long)
1680                                       le64_to_cpu(hwcqe->qp_handle));
1681         if (!qp) {
1682                 dev_err(&cq->hwq.pdev->dev,
1683                         "QPLIB: FP: Process Req qp is NULL");
1684                 return -EINVAL;
1685         }
1686         sq = &qp->sq;
1687
1688         cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
1689         if (cqe_sq_cons > sq->hwq.max_elements) {
1690                 dev_err(&cq->hwq.pdev->dev,
1691                         "QPLIB: FP: CQ Process req reported ");
1692                 dev_err(&cq->hwq.pdev->dev,
1693                         "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1694                         cqe_sq_cons, sq->hwq.max_elements);
1695                 return -EINVAL;
1696         }
1697         /* If we were in the middle of flushing the SQ, continue */
1698         if (sq->flush_in_progress)
1699                 goto flush;
1700
1701         /* Require to walk the sq's swq to fabricate CQEs for all previously
1702          * signaled SWQEs due to CQE aggregation from the current sq cons
1703          * to the cqe_sq_cons
1704          */
1705         cqe = *pcqe;
1706         while (*budget) {
1707                 sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1708                 if (sw_sq_cons == cqe_sq_cons)
1709                         /* Done */
1710                         break;
1711
1712                 swq = &sq->swq[sw_sq_cons];
1713                 memset(cqe, 0, sizeof(*cqe));
1714                 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1715                 cqe->qp_handle = (u64)(unsigned long)qp;
1716                 cqe->src_qp = qp->id;
1717                 cqe->wr_id = swq->wr_id;
1718                 if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID)
1719                         goto skip;
1720                 cqe->type = swq->type;
1721
1722                 /* For the last CQE, check for status.  For errors, regardless
1723                  * of the request being signaled or not, it must complete with
1724                  * the hwcqe error status
1725                  */
1726                 if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons &&
1727                     hwcqe->status != CQ_REQ_STATUS_OK) {
1728                         cqe->status = hwcqe->status;
1729                         dev_err(&cq->hwq.pdev->dev,
1730                                 "QPLIB: FP: CQ Processed Req ");
1731                         dev_err(&cq->hwq.pdev->dev,
1732                                 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x",
1733                                 sw_sq_cons, cqe->wr_id, cqe->status);
1734                         cqe++;
1735                         (*budget)--;
1736                         sq->flush_in_progress = true;
1737                         /* Must block new posting of SQ and RQ */
1738                         qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
1739                         sq->condition = false;
1740                         sq->single = false;
1741                 } else {
1742                         if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
1743                                 /* Before we complete, do WA 9060 */
1744                                 if (do_wa9060(qp, cq, cq_cons, sw_sq_cons,
1745                                               cqe_sq_cons)) {
1746                                         *lib_qp = qp;
1747                                         goto out;
1748                                 }
1749                                 cqe->status = CQ_REQ_STATUS_OK;
1750                                 cqe++;
1751                                 (*budget)--;
1752                         }
1753                 }
1754 skip:
1755                 sq->hwq.cons++;
1756                 if (sq->single)
1757                         break;
1758         }
1759 out:
1760         *pcqe = cqe;
1761         if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) {
1762                 /* Out of budget */
1763                 rc = -EAGAIN;
1764                 goto done;
1765         }
1766         /*
1767          * Back to normal completion mode only after it has completed all of
1768          * the WC for this CQE
1769          */
1770         sq->single = false;
1771         if (!sq->flush_in_progress)
1772                 goto done;
1773 flush:
1774         /* Require to walk the sq's swq to fabricate CQEs for all
1775          * previously posted SWQEs due to the error CQE received
1776          */
1777         rc = __flush_sq(sq, qp, pcqe, budget);
1778         if (!rc)
1779                 sq->flush_in_progress = false;
1780 done:
1781         return rc;
1782 }
1783
1784 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
1785                                         struct cq_res_rc *hwcqe,
1786                                         struct bnxt_qplib_cqe **pcqe,
1787                                         int *budget)
1788 {
1789         struct bnxt_qplib_qp *qp;
1790         struct bnxt_qplib_q *rq;
1791         struct bnxt_qplib_cqe *cqe;
1792         u32 wr_id_idx;
1793         int rc = 0;
1794
1795         qp = (struct bnxt_qplib_qp *)((unsigned long)
1796                                       le64_to_cpu(hwcqe->qp_handle));
1797         if (!qp) {
1798                 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL");
1799                 return -EINVAL;
1800         }
1801         cqe = *pcqe;
1802         cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1803         cqe->length = le32_to_cpu(hwcqe->length);
1804         cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);
1805         cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);
1806         cqe->flags = le16_to_cpu(hwcqe->flags);
1807         cqe->status = hwcqe->status;
1808         cqe->qp_handle = (u64)(unsigned long)qp;
1809
1810         wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
1811                                 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
1812         rq = &qp->rq;
1813         if (wr_id_idx > rq->hwq.max_elements) {
1814                 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC ");
1815                 dev_err(&cq->hwq.pdev->dev,
1816                         "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
1817                         wr_id_idx, rq->hwq.max_elements);
1818                 return -EINVAL;
1819         }
1820         if (rq->flush_in_progress)
1821                 goto flush_rq;
1822
1823         cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1824         cqe++;
1825         (*budget)--;
1826         rq->hwq.cons++;
1827         *pcqe = cqe;
1828
1829         if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1830                 rq->flush_in_progress = true;
1831 flush_rq:
1832                 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RC, pcqe, budget);
1833                 if (!rc)
1834                         rq->flush_in_progress = false;
1835         }
1836         return rc;
1837 }
1838
1839 static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
1840                                         struct cq_res_ud *hwcqe,
1841                                         struct bnxt_qplib_cqe **pcqe,
1842                                         int *budget)
1843 {
1844         struct bnxt_qplib_qp *qp;
1845         struct bnxt_qplib_q *rq;
1846         struct bnxt_qplib_cqe *cqe;
1847         u32 wr_id_idx;
1848         int rc = 0;
1849
1850         qp = (struct bnxt_qplib_qp *)((unsigned long)
1851                                       le64_to_cpu(hwcqe->qp_handle));
1852         if (!qp) {
1853                 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL");
1854                 return -EINVAL;
1855         }
1856         cqe = *pcqe;
1857         cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1858         cqe->length = le32_to_cpu(hwcqe->length);
1859         cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
1860         cqe->flags = le16_to_cpu(hwcqe->flags);
1861         cqe->status = hwcqe->status;
1862         cqe->qp_handle = (u64)(unsigned long)qp;
1863         memcpy(cqe->smac, hwcqe->src_mac, 6);
1864         wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
1865                                 & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK;
1866         cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |
1867                                   ((le32_to_cpu(
1868                                   hwcqe->src_qp_high_srq_or_rq_wr_id) &
1869                                  CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8);
1870
1871         rq = &qp->rq;
1872         if (wr_id_idx > rq->hwq.max_elements) {
1873                 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD ");
1874                 dev_err(&cq->hwq.pdev->dev,
1875                         "QPLIB: wr_id idx %#x exceeded RQ max %#x",
1876                         wr_id_idx, rq->hwq.max_elements);
1877                 return -EINVAL;
1878         }
1879         if (rq->flush_in_progress)
1880                 goto flush_rq;
1881
1882         cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1883         cqe++;
1884         (*budget)--;
1885         rq->hwq.cons++;
1886         *pcqe = cqe;
1887
1888         if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1889                 rq->flush_in_progress = true;
1890 flush_rq:
1891                 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_UD, pcqe, budget);
1892                 if (!rc)
1893                         rq->flush_in_progress = false;
1894         }
1895         return rc;
1896 }
1897
1898 bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
1899 {
1900         struct cq_base *hw_cqe, **hw_cqe_ptr;
1901         unsigned long flags;
1902         u32 sw_cons, raw_cons;
1903         bool rc = true;
1904
1905         spin_lock_irqsave(&cq->hwq.lock, flags);
1906         raw_cons = cq->hwq.cons;
1907         sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
1908         hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
1909         hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
1910
1911          /* Check for Valid bit. If the CQE is valid, return false */
1912         rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements);
1913         spin_unlock_irqrestore(&cq->hwq.lock, flags);
1914         return rc;
1915 }
1916
1917 static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
1918                                                 struct cq_res_raweth_qp1 *hwcqe,
1919                                                 struct bnxt_qplib_cqe **pcqe,
1920                                                 int *budget)
1921 {
1922         struct bnxt_qplib_qp *qp;
1923         struct bnxt_qplib_q *rq;
1924         struct bnxt_qplib_cqe *cqe;
1925         u32 wr_id_idx;
1926         int rc = 0;
1927
1928         qp = (struct bnxt_qplib_qp *)((unsigned long)
1929                                       le64_to_cpu(hwcqe->qp_handle));
1930         if (!qp) {
1931                 dev_err(&cq->hwq.pdev->dev,
1932                         "QPLIB: process_cq Raw/QP1 qp is NULL");
1933                 return -EINVAL;
1934         }
1935         cqe = *pcqe;
1936         cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1937         cqe->flags = le16_to_cpu(hwcqe->flags);
1938         cqe->qp_handle = (u64)(unsigned long)qp;
1939
1940         wr_id_idx =
1941                 le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)
1942                                 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK;
1943         cqe->src_qp = qp->id;
1944         if (qp->id == 1 && !cqe->length) {
1945                 /* Add workaround for the length misdetection */
1946                 cqe->length = 296;
1947         } else {
1948                 cqe->length = le16_to_cpu(hwcqe->length);
1949         }
1950         cqe->pkey_index = qp->pkey_index;
1951         memcpy(cqe->smac, qp->smac, 6);
1952
1953         cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);
1954         cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
1955
1956         rq = &qp->rq;
1957         if (wr_id_idx > rq->hwq.max_elements) {
1958                 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
1959                 dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x",
1960                         wr_id_idx, rq->hwq.max_elements);
1961                 return -EINVAL;
1962         }
1963         if (rq->flush_in_progress)
1964                 goto flush_rq;
1965
1966         cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1967         cqe++;
1968         (*budget)--;
1969         rq->hwq.cons++;
1970         *pcqe = cqe;
1971
1972         if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1973                 rq->flush_in_progress = true;
1974 flush_rq:
1975                 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RAWETH_QP1, pcqe,
1976                                 budget);
1977                 if (!rc)
1978                         rq->flush_in_progress = false;
1979         }
1980         return rc;
1981 }
1982
1983 static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
1984                                           struct cq_terminal *hwcqe,
1985                                           struct bnxt_qplib_cqe **pcqe,
1986                                           int *budget)
1987 {
1988         struct bnxt_qplib_qp *qp;
1989         struct bnxt_qplib_q *sq, *rq;
1990         struct bnxt_qplib_cqe *cqe;
1991         u32 sw_cons = 0, cqe_cons;
1992         int rc = 0;
1993         u8 opcode = 0;
1994
1995         /* Check the Status */
1996         if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
1997                 dev_warn(&cq->hwq.pdev->dev,
1998                          "QPLIB: FP: CQ Process Terminal Error status = 0x%x",
1999                          hwcqe->status);
2000
2001         qp = (struct bnxt_qplib_qp *)((unsigned long)
2002                                       le64_to_cpu(hwcqe->qp_handle));
2003         if (!qp) {
2004                 dev_err(&cq->hwq.pdev->dev,
2005                         "QPLIB: FP: CQ Process terminal qp is NULL");
2006                 return -EINVAL;
2007         }
2008         /* Must block new posting of SQ and RQ */
2009         qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2010
2011         sq = &qp->sq;
2012         rq = &qp->rq;
2013
2014         cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
2015         if (cqe_cons == 0xFFFF)
2016                 goto do_rq;
2017
2018         if (cqe_cons > sq->hwq.max_elements) {
2019                 dev_err(&cq->hwq.pdev->dev,
2020                         "QPLIB: FP: CQ Process terminal reported ");
2021                 dev_err(&cq->hwq.pdev->dev,
2022                         "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
2023                         cqe_cons, sq->hwq.max_elements);
2024                 goto do_rq;
2025         }
2026         /* If we were in the middle of flushing, continue */
2027         if (sq->flush_in_progress)
2028                 goto flush_sq;
2029
2030         /* Terminal CQE can also include aggregated successful CQEs prior.
2031          * So we must complete all CQEs from the current sq's cons to the
2032          * cq_cons with status OK
2033          */
2034         cqe = *pcqe;
2035         while (*budget) {
2036                 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2037                 if (sw_cons == cqe_cons)
2038                         break;
2039                 if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2040                         memset(cqe, 0, sizeof(*cqe));
2041                         cqe->status = CQ_REQ_STATUS_OK;
2042                         cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2043                         cqe->qp_handle = (u64)(unsigned long)qp;
2044                         cqe->src_qp = qp->id;
2045                         cqe->wr_id = sq->swq[sw_cons].wr_id;
2046                         cqe->type = sq->swq[sw_cons].type;
2047                         cqe++;
2048                         (*budget)--;
2049                 }
2050                 sq->hwq.cons++;
2051         }
2052         *pcqe = cqe;
2053         if (!(*budget) && sw_cons != cqe_cons) {
2054                 /* Out of budget */
2055                 rc = -EAGAIN;
2056                 goto sq_done;
2057         }
2058         sq->flush_in_progress = true;
2059 flush_sq:
2060         rc = __flush_sq(sq, qp, pcqe, budget);
2061         if (!rc)
2062                 sq->flush_in_progress = false;
2063 sq_done:
2064         if (rc)
2065                 return rc;
2066 do_rq:
2067         cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
2068         if (cqe_cons == 0xFFFF) {
2069                 goto done;
2070         } else if (cqe_cons > rq->hwq.max_elements) {
2071                 dev_err(&cq->hwq.pdev->dev,
2072                         "QPLIB: FP: CQ Processed terminal ");
2073                 dev_err(&cq->hwq.pdev->dev,
2074                         "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x",
2075                         cqe_cons, rq->hwq.max_elements);
2076                 goto done;
2077         }
2078         /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
2079          * from the current rq->cons to the rq->prod regardless what the
2080          * rq->cons the terminal CQE indicates
2081          */
2082         rq->flush_in_progress = true;
2083         switch (qp->type) {
2084         case CMDQ_CREATE_QP1_TYPE_GSI:
2085                 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
2086                 break;
2087         case CMDQ_CREATE_QP_TYPE_RC:
2088                 opcode = CQ_BASE_CQE_TYPE_RES_RC;
2089                 break;
2090         case CMDQ_CREATE_QP_TYPE_UD:
2091                 opcode = CQ_BASE_CQE_TYPE_RES_UD;
2092                 break;
2093         }
2094
2095         rc = __flush_rq(rq, qp, opcode, pcqe, budget);
2096         if (!rc)
2097                 rq->flush_in_progress = false;
2098 done:
2099         return rc;
2100 }
2101
2102 static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
2103                                         struct cq_cutoff *hwcqe)
2104 {
2105         /* Check the Status */
2106         if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
2107                 dev_err(&cq->hwq.pdev->dev,
2108                         "QPLIB: FP: CQ Process Cutoff Error status = 0x%x",
2109                         hwcqe->status);
2110                 return -EINVAL;
2111         }
2112         clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
2113         wake_up_interruptible(&cq->waitq);
2114
2115         return 0;
2116 }
2117
2118 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
2119                        int num_cqes, struct bnxt_qplib_qp **lib_qp)
2120 {
2121         struct cq_base *hw_cqe, **hw_cqe_ptr;
2122         unsigned long flags;
2123         u32 sw_cons, raw_cons;
2124         int budget, rc = 0;
2125
2126         spin_lock_irqsave(&cq->hwq.lock, flags);
2127         raw_cons = cq->hwq.cons;
2128         budget = num_cqes;
2129
2130         while (budget) {
2131                 sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2132                 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2133                 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2134
2135                 /* Check for Valid bit */
2136                 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
2137                         break;
2138
2139                 /* From the device's respective CQE format to qplib_wc*/
2140                 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
2141                 case CQ_BASE_CQE_TYPE_REQ:
2142                         rc = bnxt_qplib_cq_process_req(cq,
2143                                                        (struct cq_req *)hw_cqe,
2144                                                        &cqe, &budget,
2145                                                        sw_cons, lib_qp);
2146                         break;
2147                 case CQ_BASE_CQE_TYPE_RES_RC:
2148                         rc = bnxt_qplib_cq_process_res_rc(cq,
2149                                                           (struct cq_res_rc *)
2150                                                           hw_cqe, &cqe,
2151                                                           &budget);
2152                         break;
2153                 case CQ_BASE_CQE_TYPE_RES_UD:
2154                         rc = bnxt_qplib_cq_process_res_ud
2155                                         (cq, (struct cq_res_ud *)hw_cqe, &cqe,
2156                                          &budget);
2157                         break;
2158                 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2159                         rc = bnxt_qplib_cq_process_res_raweth_qp1
2160                                         (cq, (struct cq_res_raweth_qp1 *)
2161                                          hw_cqe, &cqe, &budget);
2162                         break;
2163                 case CQ_BASE_CQE_TYPE_TERMINAL:
2164                         rc = bnxt_qplib_cq_process_terminal
2165                                         (cq, (struct cq_terminal *)hw_cqe,
2166                                          &cqe, &budget);
2167                         break;
2168                 case CQ_BASE_CQE_TYPE_CUT_OFF:
2169                         bnxt_qplib_cq_process_cutoff
2170                                         (cq, (struct cq_cutoff *)hw_cqe);
2171                         /* Done processing this CQ */
2172                         goto exit;
2173                 default:
2174                         dev_err(&cq->hwq.pdev->dev,
2175                                 "QPLIB: process_cq unknown type 0x%lx",
2176                                 hw_cqe->cqe_type_toggle &
2177                                 CQ_BASE_CQE_TYPE_MASK);
2178                         rc = -EINVAL;
2179                         break;
2180                 }
2181                 if (rc < 0) {
2182                         if (rc == -EAGAIN)
2183                                 break;
2184                         /* Error while processing the CQE, just skip to the
2185                          * next one
2186                          */
2187                         dev_err(&cq->hwq.pdev->dev,
2188                                 "QPLIB: process_cqe error rc = 0x%x", rc);
2189                 }
2190                 raw_cons++;
2191         }
2192         if (cq->hwq.cons != raw_cons) {
2193                 cq->hwq.cons = raw_cons;
2194                 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ);
2195         }
2196 exit:
2197         spin_unlock_irqrestore(&cq->hwq.lock, flags);
2198         return num_cqes - budget;
2199 }
2200
2201 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
2202 {
2203         unsigned long flags;
2204
2205         spin_lock_irqsave(&cq->hwq.lock, flags);
2206         if (arm_type)
2207                 bnxt_qplib_arm_cq(cq, arm_type);
2208
2209         spin_unlock_irqrestore(&cq->hwq.lock, flags);
2210 }