drivers/net/ethernet/qlogic/qed/qed_spq.c

   1 /* QLogic qed NIC Driver
   2  * Copyright (c) 2015-2017  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/types.h>
  34 #include <asm/byteorder.h>
  35 #include <linux/io.h>
  36 #include <linux/delay.h>
  37 #include <linux/dma-mapping.h>
  38 #include <linux/errno.h>
  39 #include <linux/kernel.h>
  40 #include <linux/list.h>
  41 #include <linux/pci.h>
  42 #include <linux/slab.h>
  43 #include <linux/spinlock.h>
  44 #include <linux/string.h>
  45 #include "qed.h"
  46 #include "qed_cxt.h"
  47 #include "qed_dev_api.h"
  48 #include "qed_hsi.h"
  49 #include "qed_hw.h"
  50 #include "qed_int.h"
  51 #include "qed_iscsi.h"
  52 #include "qed_mcp.h"
  53 #include "qed_ooo.h"
  54 #include "qed_reg_addr.h"
  55 #include "qed_sp.h"
  56 #include "qed_sriov.h"
  57 #include "qed_rdma.h"
  58
  59 /***************************************************************************
  60 * Structures & Definitions
  61 ***************************************************************************/
  62
  63 #define SPQ_HIGH_PRI_RESERVE_DEFAULT    (1)
  64
  65 #define SPQ_BLOCK_DELAY_MAX_ITER        (10)
  66 #define SPQ_BLOCK_DELAY_US              (10)
  67 #define SPQ_BLOCK_SLEEP_MAX_ITER        (1000)
  68 #define SPQ_BLOCK_SLEEP_MS              (5)
  69
  70 /***************************************************************************
  71 * Blocking Imp. (BLOCK/EBLOCK mode)
  72 ***************************************************************************/
  73 static void qed_spq_blocking_cb(struct qed_hwfn *p_hwfn,
  74                                 void *cookie,
  75                                 union event_ring_data *data, u8 fw_return_code)
  76 {
  77         struct qed_spq_comp_done *comp_done;
  78
  79         comp_done = (struct qed_spq_comp_done *)cookie;
  80
  81         comp_done->fw_return_code = fw_return_code;
  82
  83         /* Make sure completion done is visible on waiting thread */
  84         smp_store_release(&comp_done->done, 0x1);
  85 }
  86
  87 static int __qed_spq_block(struct qed_hwfn *p_hwfn,
  88                            struct qed_spq_entry *p_ent,
  89                            u8 *p_fw_ret, bool sleep_between_iter)
  90 {
  91         struct qed_spq_comp_done *comp_done;
  92         u32 iter_cnt;
  93
  94         comp_done = (struct qed_spq_comp_done *)p_ent->comp_cb.cookie;
  95         iter_cnt = sleep_between_iter ? SPQ_BLOCK_SLEEP_MAX_ITER
  96                                       : SPQ_BLOCK_DELAY_MAX_ITER;
  97
  98         while (iter_cnt--) {
  99                 /* Validate we receive completion update */
 100                 if (READ_ONCE(comp_done->done) == 1) {
 101                         /* Read updated FW return value */
 102                         smp_read_barrier_depends();
 103                         if (p_fw_ret)
 104                                 *p_fw_ret = comp_done->fw_return_code;
 105                         return 0;
 106                 }
 107
 108                 if (sleep_between_iter)
 109                         msleep(SPQ_BLOCK_SLEEP_MS);
 110                 else
 111                         udelay(SPQ_BLOCK_DELAY_US);
 112         }
 113
 114         return -EBUSY;
 115 }
 116
 117 static int qed_spq_block(struct qed_hwfn *p_hwfn,
 118                          struct qed_spq_entry *p_ent,
 119                          u8 *p_fw_ret, bool skip_quick_poll)
 120 {
 121         struct qed_spq_comp_done *comp_done;
 122         struct qed_ptt *p_ptt;
 123         int rc;
 124
 125         /* A relatively short polling period w/o sleeping, to allow the FW to
 126          * complete the ramrod and thus possibly to avoid the following sleeps.
 127          */
 128         if (!skip_quick_poll) {
 129                 rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, false);
 130                 if (!rc)
 131                         return 0;
 132         }
 133
 134         /* Move to polling with a sleeping period between iterations */
 135         rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, true);
 136         if (!rc)
 137                 return 0;
 138
 139         p_ptt = qed_ptt_acquire(p_hwfn);
 140         if (!p_ptt) {
 141                 DP_NOTICE(p_hwfn, "ptt, failed to acquire\n");
 142                 return -EAGAIN;
 143         }
 144
 145         DP_INFO(p_hwfn, "Ramrod is stuck, requesting MCP drain\n");
 146         rc = qed_mcp_drain(p_hwfn, p_ptt);
 147         if (rc) {
 148                 DP_NOTICE(p_hwfn, "MCP drain failed\n");
 149                 goto err;
 150         }
 151
 152         /* Retry after drain */
 153         rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, true);
 154         if (!rc)
 155                 goto out;
 156
 157         comp_done = (struct qed_spq_comp_done *)p_ent->comp_cb.cookie;
 158         if (comp_done->done == 1)
 159                 if (p_fw_ret)
 160                         *p_fw_ret = comp_done->fw_return_code;
 161 out:
 162         qed_ptt_release(p_hwfn, p_ptt);
 163         return 0;
 164
 165 err:
 166         qed_ptt_release(p_hwfn, p_ptt);
 167         DP_NOTICE(p_hwfn,
 168                   "Ramrod is stuck [CID %08x cmd %02x protocol %02x echo %04x]\n",
 169                   le32_to_cpu(p_ent->elem.hdr.cid),
 170                   p_ent->elem.hdr.cmd_id,
 171                   p_ent->elem.hdr.protocol_id,
 172                   le16_to_cpu(p_ent->elem.hdr.echo));
 173
 174         return -EBUSY;
 175 }
 176
 177 /***************************************************************************
 178 * SPQ entries inner API
 179 ***************************************************************************/
 180 static int qed_spq_fill_entry(struct qed_hwfn *p_hwfn,
 181                               struct qed_spq_entry *p_ent)
 182 {
 183         p_ent->flags = 0;
 184
 185         switch (p_ent->comp_mode) {
 186         case QED_SPQ_MODE_EBLOCK:
 187         case QED_SPQ_MODE_BLOCK:
 188                 p_ent->comp_cb.function = qed_spq_blocking_cb;
 189                 break;
 190         case QED_SPQ_MODE_CB:
 191                 break;
 192         default:
 193                 DP_NOTICE(p_hwfn, "Unknown SPQE completion mode %d\n",
 194                           p_ent->comp_mode);
 195                 return -EINVAL;
 196         }
 197
 198         DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
 199                    "Ramrod header: [CID 0x%08x CMD 0x%02x protocol 0x%02x] Data pointer: [%08x:%08x] Completion Mode: %s\n",
 200                    p_ent->elem.hdr.cid,
 201                    p_ent->elem.hdr.cmd_id,
 202                    p_ent->elem.hdr.protocol_id,
 203                    p_ent->elem.data_ptr.hi,
 204                    p_ent->elem.data_ptr.lo,
 205                    D_TRINE(p_ent->comp_mode, QED_SPQ_MODE_EBLOCK,
 206                            QED_SPQ_MODE_BLOCK, "MODE_EBLOCK", "MODE_BLOCK",
 207                            "MODE_CB"));
 208
 209         return 0;
 210 }
 211
 212 /***************************************************************************
 213 * HSI access
 214 ***************************************************************************/
 215 static void qed_spq_hw_initialize(struct qed_hwfn *p_hwfn,
 216                                   struct qed_spq *p_spq)
 217 {
 218         struct core_conn_context *p_cxt;
 219         struct qed_cxt_info cxt_info;
 220         u16 physical_q;
 221         int rc;
 222
 223         cxt_info.iid = p_spq->cid;
 224
 225         rc = qed_cxt_get_cid_info(p_hwfn, &cxt_info);
 226
 227         if (rc < 0) {
 228                 DP_NOTICE(p_hwfn, "Cannot find context info for cid=%d\n",
 229                           p_spq->cid);
 230                 return;
 231         }
 232
 233         p_cxt = cxt_info.p_cxt;
 234
 235         SET_FIELD(p_cxt->xstorm_ag_context.flags10,
 236                   XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN, 1);
 237         SET_FIELD(p_cxt->xstorm_ag_context.flags1,
 238                   XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE, 1);
 239         SET_FIELD(p_cxt->xstorm_ag_context.flags9,
 240                   XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN, 1);
 241
 242         /* QM physical queue */
 243         physical_q = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LB);
 244         p_cxt->xstorm_ag_context.physical_q0 = cpu_to_le16(physical_q);
 245
 246         p_cxt->xstorm_st_context.spq_base_lo =
 247                 DMA_LO_LE(p_spq->chain.p_phys_addr);
 248         p_cxt->xstorm_st_context.spq_base_hi =
 249                 DMA_HI_LE(p_spq->chain.p_phys_addr);
 250
 251         DMA_REGPAIR_LE(p_cxt->xstorm_st_context.consolid_base_addr,
 252                        p_hwfn->p_consq->chain.p_phys_addr);
 253 }
 254
 255 static int qed_spq_hw_post(struct qed_hwfn *p_hwfn,
 256                            struct qed_spq *p_spq, struct qed_spq_entry *p_ent)
 257 {
 258         struct qed_chain *p_chain = &p_hwfn->p_spq->chain;
 259         u16 echo = qed_chain_get_prod_idx(p_chain);
 260         struct slow_path_element        *elem;
 261         struct core_db_data             db;
 262
 263         p_ent->elem.hdr.echo    = cpu_to_le16(echo);
 264         elem = qed_chain_produce(p_chain);
 265         if (!elem) {
 266                 DP_NOTICE(p_hwfn, "Failed to produce from SPQ chain\n");
 267                 return -EINVAL;
 268         }
 269
 270         *elem = p_ent->elem; /* struct assignment */
 271
 272         /* send a doorbell on the slow hwfn session */
 273         memset(&db, 0, sizeof(db));
 274         SET_FIELD(db.params, CORE_DB_DATA_DEST, DB_DEST_XCM);
 275         SET_FIELD(db.params, CORE_DB_DATA_AGG_CMD, DB_AGG_CMD_SET);
 276         SET_FIELD(db.params, CORE_DB_DATA_AGG_VAL_SEL,
 277                   DQ_XCM_CORE_SPQ_PROD_CMD);
 278         db.agg_flags = DQ_XCM_CORE_DQ_CF_CMD;
 279         db.spq_prod = cpu_to_le16(qed_chain_get_prod_idx(p_chain));
 280
 281         /* make sure the SPQE is updated before the doorbell */
 282         wmb();
 283
 284         DOORBELL(p_hwfn, qed_db_addr(p_spq->cid, DQ_DEMS_LEGACY), *(u32 *)&db);
 285
 286         /* make sure doorbell is rang */
 287         wmb();
 288
 289         DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
 290                    "Doorbelled [0x%08x, CID 0x%08x] with Flags: %02x agg_params: %02x, prod: %04x\n",
 291                    qed_db_addr(p_spq->cid, DQ_DEMS_LEGACY),
 292                    p_spq->cid, db.params, db.agg_flags,
 293                    qed_chain_get_prod_idx(p_chain));
 294
 295         return 0;
 296 }
 297
 298 /***************************************************************************
 299 * Asynchronous events
 300 ***************************************************************************/
 301 static int
 302 qed_async_event_completion(struct qed_hwfn *p_hwfn,
 303                            struct event_ring_entry *p_eqe)
 304 {
 305         qed_spq_async_comp_cb cb;
 306
 307         if (!p_hwfn->p_spq || (p_eqe->protocol_id >= MAX_PROTOCOL_TYPE))
 308                 return -EINVAL;
 309
 310         cb = p_hwfn->p_spq->async_comp_cb[p_eqe->protocol_id];
 311         if (cb) {
 312                 return cb(p_hwfn, p_eqe->opcode, p_eqe->echo,
 313                           &p_eqe->data, p_eqe->fw_return_code);
 314         } else {
 315                 DP_NOTICE(p_hwfn,
 316                           "Unknown Async completion for protocol: %d\n",
 317                           p_eqe->protocol_id);
 318                 return -EINVAL;
 319         }
 320 }
 321
 322 int
 323 qed_spq_register_async_cb(struct qed_hwfn *p_hwfn,
 324                           enum protocol_type protocol_id,
 325                           qed_spq_async_comp_cb cb)
 326 {
 327         if (!p_hwfn->p_spq || (protocol_id >= MAX_PROTOCOL_TYPE))
 328                 return -EINVAL;
 329
 330         p_hwfn->p_spq->async_comp_cb[protocol_id] = cb;
 331         return 0;
 332 }
 333
 334 void
 335 qed_spq_unregister_async_cb(struct qed_hwfn *p_hwfn,
 336                             enum protocol_type protocol_id)
 337 {
 338         if (!p_hwfn->p_spq || (protocol_id >= MAX_PROTOCOL_TYPE))
 339                 return;
 340
 341         p_hwfn->p_spq->async_comp_cb[protocol_id] = NULL;
 342 }
 343
 344 /***************************************************************************
 345 * EQ API
 346 ***************************************************************************/
 347 void qed_eq_prod_update(struct qed_hwfn *p_hwfn, u16 prod)
 348 {
 349         u32 addr = GTT_BAR0_MAP_REG_USDM_RAM +
 350                    USTORM_EQE_CONS_OFFSET(p_hwfn->rel_pf_id);
 351
 352         REG_WR16(p_hwfn, addr, prod);
 353
 354         /* keep prod updates ordered */
 355         mmiowb();
 356 }
 357
 358 int qed_eq_completion(struct qed_hwfn *p_hwfn, void *cookie)
 359 {
 360         struct qed_eq *p_eq = cookie;
 361         struct qed_chain *p_chain = &p_eq->chain;
 362         int rc = 0;
 363
 364         /* take a snapshot of the FW consumer */
 365         u16 fw_cons_idx = le16_to_cpu(*p_eq->p_fw_cons);
 366
 367         DP_VERBOSE(p_hwfn, QED_MSG_SPQ, "fw_cons_idx %x\n", fw_cons_idx);
 368
 369         /* Need to guarantee the fw_cons index we use points to a usuable
 370          * element (to comply with our chain), so our macros would comply
 371          */
 372         if ((fw_cons_idx & qed_chain_get_usable_per_page(p_chain)) ==
 373             qed_chain_get_usable_per_page(p_chain))
 374                 fw_cons_idx += qed_chain_get_unusable_per_page(p_chain);
 375
 376         /* Complete current segment of eq entries */
 377         while (fw_cons_idx != qed_chain_get_cons_idx(p_chain)) {
 378                 struct event_ring_entry *p_eqe = qed_chain_consume(p_chain);
 379
 380                 if (!p_eqe) {
 381                         rc = -EINVAL;
 382                         break;
 383                 }
 384
 385                 DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
 386                            "op %x prot %x res0 %x echo %x fwret %x flags %x\n",
 387                            p_eqe->opcode,
 388                            p_eqe->protocol_id,
 389                            p_eqe->reserved0,
 390                            le16_to_cpu(p_eqe->echo),
 391                            p_eqe->fw_return_code,
 392                            p_eqe->flags);
 393
 394                 if (GET_FIELD(p_eqe->flags, EVENT_RING_ENTRY_ASYNC)) {
 395                         if (qed_async_event_completion(p_hwfn, p_eqe))
 396                                 rc = -EINVAL;
 397                 } else if (qed_spq_completion(p_hwfn,
 398                                               p_eqe->echo,
 399                                               p_eqe->fw_return_code,
 400                                               &p_eqe->data)) {
 401                         rc = -EINVAL;
 402                 }
 403
 404                 qed_chain_recycle_consumed(p_chain);
 405         }
 406
 407         qed_eq_prod_update(p_hwfn, qed_chain_get_prod_idx(p_chain));
 408
 409         return rc;
 410 }
 411
 412 int qed_eq_alloc(struct qed_hwfn *p_hwfn, u16 num_elem)
 413 {
 414         struct qed_eq *p_eq;
 415
 416         /* Allocate EQ struct */
 417         p_eq = kzalloc(sizeof(*p_eq), GFP_KERNEL);
 418         if (!p_eq)
 419                 return -ENOMEM;
 420
 421         /* Allocate and initialize EQ chain*/
 422         if (qed_chain_alloc(p_hwfn->cdev,
 423                             QED_CHAIN_USE_TO_PRODUCE,
 424                             QED_CHAIN_MODE_PBL,
 425                             QED_CHAIN_CNT_TYPE_U16,
 426                             num_elem,
 427                             sizeof(union event_ring_element),
 428                             &p_eq->chain, NULL))
 429                 goto eq_allocate_fail;
 430
 431         /* register EQ completion on the SP SB */
 432         qed_int_register_cb(p_hwfn, qed_eq_completion,
 433                             p_eq, &p_eq->eq_sb_index, &p_eq->p_fw_cons);
 434
 435         p_hwfn->p_eq = p_eq;
 436         return 0;
 437
 438 eq_allocate_fail:
 439         kfree(p_eq);
 440         return -ENOMEM;
 441 }
 442
 443 void qed_eq_setup(struct qed_hwfn *p_hwfn)
 444 {
 445         qed_chain_reset(&p_hwfn->p_eq->chain);
 446 }
 447
 448 void qed_eq_free(struct qed_hwfn *p_hwfn)
 449 {
 450         if (!p_hwfn->p_eq)
 451                 return;
 452
 453         qed_chain_free(p_hwfn->cdev, &p_hwfn->p_eq->chain);
 454
 455         kfree(p_hwfn->p_eq);
 456         p_hwfn->p_eq = NULL;
 457 }
 458
 459 /***************************************************************************
 460 * CQE API - manipulate EQ functionality
 461 ***************************************************************************/
 462 static int qed_cqe_completion(struct qed_hwfn *p_hwfn,
 463                               struct eth_slow_path_rx_cqe *cqe,
 464                               enum protocol_type protocol)
 465 {
 466         if (IS_VF(p_hwfn->cdev))
 467                 return 0;
 468
 469         /* @@@tmp - it's possible we'll eventually want to handle some
 470          * actual commands that can arrive here, but for now this is only
 471          * used to complete the ramrod using the echo value on the cqe
 472          */
 473         return qed_spq_completion(p_hwfn, cqe->echo, 0, NULL);
 474 }
 475
 476 int qed_eth_cqe_completion(struct qed_hwfn *p_hwfn,
 477                            struct eth_slow_path_rx_cqe *cqe)
 478 {
 479         int rc;
 480
 481         rc = qed_cqe_completion(p_hwfn, cqe, PROTOCOLID_ETH);
 482         if (rc)
 483                 DP_NOTICE(p_hwfn,
 484                           "Failed to handle RXQ CQE [cmd 0x%02x]\n",
 485                           cqe->ramrod_cmd_id);
 486
 487         return rc;
 488 }
 489
 490 /***************************************************************************
 491 * Slow hwfn Queue (spq)
 492 ***************************************************************************/
 493 void qed_spq_setup(struct qed_hwfn *p_hwfn)
 494 {
 495         struct qed_spq *p_spq = p_hwfn->p_spq;
 496         struct qed_spq_entry *p_virt = NULL;
 497         dma_addr_t p_phys = 0;
 498         u32 i, capacity;
 499
 500         INIT_LIST_HEAD(&p_spq->pending);
 501         INIT_LIST_HEAD(&p_spq->completion_pending);
 502         INIT_LIST_HEAD(&p_spq->free_pool);
 503         INIT_LIST_HEAD(&p_spq->unlimited_pending);
 504         spin_lock_init(&p_spq->lock);
 505
 506         /* SPQ empty pool */
 507         p_phys  = p_spq->p_phys + offsetof(struct qed_spq_entry, ramrod);
 508         p_virt  = p_spq->p_virt;
 509
 510         capacity = qed_chain_get_capacity(&p_spq->chain);
 511         for (i = 0; i < capacity; i++) {
 512                 DMA_REGPAIR_LE(p_virt->elem.data_ptr, p_phys);
 513
 514                 list_add_tail(&p_virt->list, &p_spq->free_pool);
 515
 516                 p_virt++;
 517                 p_phys += sizeof(struct qed_spq_entry);
 518         }
 519
 520         /* Statistics */
 521         p_spq->normal_count             = 0;
 522         p_spq->comp_count               = 0;
 523         p_spq->comp_sent_count          = 0;
 524         p_spq->unlimited_pending_count  = 0;
 525
 526         bitmap_zero(p_spq->p_comp_bitmap, SPQ_RING_SIZE);
 527         p_spq->comp_bitmap_idx = 0;
 528
 529         /* SPQ cid, cannot fail */
 530         qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_CORE, &p_spq->cid);
 531         qed_spq_hw_initialize(p_hwfn, p_spq);
 532
 533         /* reset the chain itself */
 534         qed_chain_reset(&p_spq->chain);
 535 }
 536
 537 int qed_spq_alloc(struct qed_hwfn *p_hwfn)
 538 {
 539         struct qed_spq_entry *p_virt = NULL;
 540         struct qed_spq *p_spq = NULL;
 541         dma_addr_t p_phys = 0;
 542         u32 capacity;
 543
 544         /* SPQ struct */
 545         p_spq = kzalloc(sizeof(struct qed_spq), GFP_KERNEL);
 546         if (!p_spq)
 547                 return -ENOMEM;
 548
 549         /* SPQ ring  */
 550         if (qed_chain_alloc(p_hwfn->cdev,
 551                             QED_CHAIN_USE_TO_PRODUCE,
 552                             QED_CHAIN_MODE_SINGLE,
 553                             QED_CHAIN_CNT_TYPE_U16,
 554                             0,   /* N/A when the mode is SINGLE */
 555                             sizeof(struct slow_path_element),
 556                             &p_spq->chain, NULL))
 557                 goto spq_allocate_fail;
 558
 559         /* allocate and fill the SPQ elements (incl. ramrod data list) */
 560         capacity = qed_chain_get_capacity(&p_spq->chain);
 561         p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
 562                                     capacity * sizeof(struct qed_spq_entry),
 563                                     &p_phys, GFP_KERNEL);
 564         if (!p_virt)
 565                 goto spq_allocate_fail;
 566
 567         p_spq->p_virt = p_virt;
 568         p_spq->p_phys = p_phys;
 569         p_hwfn->p_spq = p_spq;
 570
 571         return 0;
 572
 573 spq_allocate_fail:
 574         qed_chain_free(p_hwfn->cdev, &p_spq->chain);
 575         kfree(p_spq);
 576         return -ENOMEM;
 577 }
 578
 579 void qed_spq_free(struct qed_hwfn *p_hwfn)
 580 {
 581         struct qed_spq *p_spq = p_hwfn->p_spq;
 582         u32 capacity;
 583
 584         if (!p_spq)
 585                 return;
 586
 587         if (p_spq->p_virt) {
 588                 capacity = qed_chain_get_capacity(&p_spq->chain);
 589                 dma_free_coherent(&p_hwfn->cdev->pdev->dev,
 590                                   capacity *
 591                                   sizeof(struct qed_spq_entry),
 592                                   p_spq->p_virt, p_spq->p_phys);
 593         }
 594
 595         qed_chain_free(p_hwfn->cdev, &p_spq->chain);
 596         kfree(p_spq);
 597         p_hwfn->p_spq = NULL;
 598 }
 599
 600 int qed_spq_get_entry(struct qed_hwfn *p_hwfn, struct qed_spq_entry **pp_ent)
 601 {
 602         struct qed_spq *p_spq = p_hwfn->p_spq;
 603         struct qed_spq_entry *p_ent = NULL;
 604         int rc = 0;
 605
 606         spin_lock_bh(&p_spq->lock);
 607
 608         if (list_empty(&p_spq->free_pool)) {
 609                 p_ent = kzalloc(sizeof(*p_ent), GFP_ATOMIC);
 610                 if (!p_ent) {
 611                         DP_NOTICE(p_hwfn,
 612                                   "Failed to allocate an SPQ entry for a pending ramrod\n");
 613                         rc = -ENOMEM;
 614                         goto out_unlock;
 615                 }
 616                 p_ent->queue = &p_spq->unlimited_pending;
 617         } else {
 618                 p_ent = list_first_entry(&p_spq->free_pool,
 619                                          struct qed_spq_entry, list);
 620                 list_del(&p_ent->list);
 621                 p_ent->queue = &p_spq->pending;
 622         }
 623
 624         *pp_ent = p_ent;
 625
 626 out_unlock:
 627         spin_unlock_bh(&p_spq->lock);
 628         return rc;
 629 }
 630
 631 /* Locked variant; Should be called while the SPQ lock is taken */
 632 static void __qed_spq_return_entry(struct qed_hwfn *p_hwfn,
 633                                    struct qed_spq_entry *p_ent)
 634 {
 635         list_add_tail(&p_ent->list, &p_hwfn->p_spq->free_pool);
 636 }
 637
 638 void qed_spq_return_entry(struct qed_hwfn *p_hwfn, struct qed_spq_entry *p_ent)
 639 {
 640         spin_lock_bh(&p_hwfn->p_spq->lock);
 641         __qed_spq_return_entry(p_hwfn, p_ent);
 642         spin_unlock_bh(&p_hwfn->p_spq->lock);
 643 }
 644
 645 /**
 646  * @brief qed_spq_add_entry - adds a new entry to the pending
 647  *        list. Should be used while lock is being held.
 648  *
 649  * Addes an entry to the pending list is there is room (en empty
 650  * element is available in the free_pool), or else places the
 651  * entry in the unlimited_pending pool.
 652  *
 653  * @param p_hwfn
 654  * @param p_ent
 655  * @param priority
 656  *
 657  * @return int
 658  */
 659 static int qed_spq_add_entry(struct qed_hwfn *p_hwfn,
 660                              struct qed_spq_entry *p_ent,
 661                              enum spq_priority priority)
 662 {
 663         struct qed_spq *p_spq = p_hwfn->p_spq;
 664
 665         if (p_ent->queue == &p_spq->unlimited_pending) {
 666
 667                 if (list_empty(&p_spq->free_pool)) {
 668                         list_add_tail(&p_ent->list, &p_spq->unlimited_pending);
 669                         p_spq->unlimited_pending_count++;
 670
 671                         return 0;
 672                 } else {
 673                         struct qed_spq_entry *p_en2;
 674
 675                         p_en2 = list_first_entry(&p_spq->free_pool,
 676                                                  struct qed_spq_entry, list);
 677                         list_del(&p_en2->list);
 678
 679                         /* Copy the ring element physical pointer to the new
 680                          * entry, since we are about to override the entire ring
 681                          * entry and don't want to lose the pointer.
 682                          */
 683                         p_ent->elem.data_ptr = p_en2->elem.data_ptr;
 684
 685                         *p_en2 = *p_ent;
 686
 687                         /* EBLOCK responsible to free the allocated p_ent */
 688                         if (p_ent->comp_mode != QED_SPQ_MODE_EBLOCK)
 689                                 kfree(p_ent);
 690
 691                         p_ent = p_en2;
 692                 }
 693         }
 694
 695         /* entry is to be placed in 'pending' queue */
 696         switch (priority) {
 697         case QED_SPQ_PRIORITY_NORMAL:
 698                 list_add_tail(&p_ent->list, &p_spq->pending);
 699                 p_spq->normal_count++;
 700                 break;
 701         case QED_SPQ_PRIORITY_HIGH:
 702                 list_add(&p_ent->list, &p_spq->pending);
 703                 p_spq->high_count++;
 704                 break;
 705         default:
 706                 return -EINVAL;
 707         }
 708
 709         return 0;
 710 }
 711
 712 /***************************************************************************
 713 * Accessor
 714 ***************************************************************************/
 715 u32 qed_spq_get_cid(struct qed_hwfn *p_hwfn)
 716 {
 717         if (!p_hwfn->p_spq)
 718                 return 0xffffffff;      /* illegal */
 719         return p_hwfn->p_spq->cid;
 720 }
 721
 722 /***************************************************************************
 723 * Posting new Ramrods
 724 ***************************************************************************/
 725 static int qed_spq_post_list(struct qed_hwfn *p_hwfn,
 726                              struct list_head *head, u32 keep_reserve)
 727 {
 728         struct qed_spq *p_spq = p_hwfn->p_spq;
 729         int rc;
 730
 731         while (qed_chain_get_elem_left(&p_spq->chain) > keep_reserve &&
 732                !list_empty(head)) {
 733                 struct qed_spq_entry *p_ent =
 734                         list_first_entry(head, struct qed_spq_entry, list);
 735                 list_del(&p_ent->list);
 736                 list_add_tail(&p_ent->list, &p_spq->completion_pending);
 737                 p_spq->comp_sent_count++;
 738
 739                 rc = qed_spq_hw_post(p_hwfn, p_spq, p_ent);
 740                 if (rc) {
 741                         list_del(&p_ent->list);
 742                         __qed_spq_return_entry(p_hwfn, p_ent);
 743                         return rc;
 744                 }
 745         }
 746
 747         return 0;
 748 }
 749
 750 static int qed_spq_pend_post(struct qed_hwfn *p_hwfn)
 751 {
 752         struct qed_spq *p_spq = p_hwfn->p_spq;
 753         struct qed_spq_entry *p_ent = NULL;
 754
 755         while (!list_empty(&p_spq->free_pool)) {
 756                 if (list_empty(&p_spq->unlimited_pending))
 757                         break;
 758
 759                 p_ent = list_first_entry(&p_spq->unlimited_pending,
 760                                          struct qed_spq_entry, list);
 761                 if (!p_ent)
 762                         return -EINVAL;
 763
 764                 list_del(&p_ent->list);
 765
 766                 qed_spq_add_entry(p_hwfn, p_ent, p_ent->priority);
 767         }
 768
 769         return qed_spq_post_list(p_hwfn, &p_spq->pending,
 770                                  SPQ_HIGH_PRI_RESERVE_DEFAULT);
 771 }
 772
 773 int qed_spq_post(struct qed_hwfn *p_hwfn,
 774                  struct qed_spq_entry *p_ent, u8 *fw_return_code)
 775 {
 776         int rc = 0;
 777         struct qed_spq *p_spq = p_hwfn ? p_hwfn->p_spq : NULL;
 778         bool b_ret_ent = true;
 779         bool eblock;
 780
 781         if (!p_hwfn)
 782                 return -EINVAL;
 783
 784         if (!p_ent) {
 785                 DP_NOTICE(p_hwfn, "Got a NULL pointer\n");
 786                 return -EINVAL;
 787         }
 788
 789         /* Complete the entry */
 790         rc = qed_spq_fill_entry(p_hwfn, p_ent);
 791
 792         spin_lock_bh(&p_spq->lock);
 793
 794         /* Check return value after LOCK is taken for cleaner error flow */
 795         if (rc)
 796                 goto spq_post_fail;
 797
 798         /* Check if entry is in block mode before qed_spq_add_entry,
 799          * which might kfree p_ent.
 800          */
 801         eblock = (p_ent->comp_mode == QED_SPQ_MODE_EBLOCK);
 802
 803         /* Add the request to the pending queue */
 804         rc = qed_spq_add_entry(p_hwfn, p_ent, p_ent->priority);
 805         if (rc)
 806                 goto spq_post_fail;
 807
 808         rc = qed_spq_pend_post(p_hwfn);
 809         if (rc) {
 810                 /* Since it's possible that pending failed for a different
 811                  * entry [although unlikely], the failed entry was already
 812                  * dealt with; No need to return it here.
 813                  */
 814                 b_ret_ent = false;
 815                 goto spq_post_fail;
 816         }
 817
 818         spin_unlock_bh(&p_spq->lock);
 819
 820         if (eblock) {
 821                 /* For entries in QED BLOCK mode, the completion code cannot
 822                  * perform the necessary cleanup - if it did, we couldn't
 823                  * access p_ent here to see whether it's successful or not.
 824                  * Thus, after gaining the answer perform the cleanup here.
 825                  */
 826                 rc = qed_spq_block(p_hwfn, p_ent, fw_return_code,
 827                                    p_ent->queue == &p_spq->unlimited_pending);
 828
 829                 if (p_ent->queue == &p_spq->unlimited_pending) {
 830                         /* This is an allocated p_ent which does not need to
 831                          * return to pool.
 832                          */
 833                         kfree(p_ent);
 834                         return rc;
 835                 }
 836
 837                 if (rc)
 838                         goto spq_post_fail2;
 839
 840                 /* return to pool */
 841                 qed_spq_return_entry(p_hwfn, p_ent);
 842         }
 843         return rc;
 844
 845 spq_post_fail2:
 846         spin_lock_bh(&p_spq->lock);
 847         list_del(&p_ent->list);
 848         qed_chain_return_produced(&p_spq->chain);
 849
 850 spq_post_fail:
 851         /* return to the free pool */
 852         if (b_ret_ent)
 853                 __qed_spq_return_entry(p_hwfn, p_ent);
 854         spin_unlock_bh(&p_spq->lock);
 855
 856         return rc;
 857 }
 858
 859 int qed_spq_completion(struct qed_hwfn *p_hwfn,
 860                        __le16 echo,
 861                        u8 fw_return_code,
 862                        union event_ring_data *p_data)
 863 {
 864         struct qed_spq          *p_spq;
 865         struct qed_spq_entry    *p_ent = NULL;
 866         struct qed_spq_entry    *tmp;
 867         struct qed_spq_entry    *found = NULL;
 868         int                     rc;
 869
 870         if (!p_hwfn)
 871                 return -EINVAL;
 872
 873         p_spq = p_hwfn->p_spq;
 874         if (!p_spq)
 875                 return -EINVAL;
 876
 877         spin_lock_bh(&p_spq->lock);
 878         list_for_each_entry_safe(p_ent, tmp, &p_spq->completion_pending, list) {
 879                 if (p_ent->elem.hdr.echo == echo) {
 880                         u16 pos = le16_to_cpu(echo) % SPQ_RING_SIZE;
 881
 882                         list_del(&p_ent->list);
 883
 884                         /* Avoid overriding of SPQ entries when getting
 885                          * out-of-order completions, by marking the completions
 886                          * in a bitmap and increasing the chain consumer only
 887                          * for the first successive completed entries.
 888                          */
 889                         __set_bit(pos, p_spq->p_comp_bitmap);
 890
 891                         while (test_bit(p_spq->comp_bitmap_idx,
 892                                         p_spq->p_comp_bitmap)) {
 893                                 __clear_bit(p_spq->comp_bitmap_idx,
 894                                             p_spq->p_comp_bitmap);
 895                                 p_spq->comp_bitmap_idx++;
 896                                 qed_chain_return_produced(&p_spq->chain);
 897                         }
 898
 899                         p_spq->comp_count++;
 900                         found = p_ent;
 901                         break;
 902                 }
 903
 904                 /* This is relatively uncommon - depends on scenarios
 905                  * which have mutliple per-PF sent ramrods.
 906                  */
 907                 DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
 908                            "Got completion for echo %04x - doesn't match echo %04x in completion pending list\n",
 909                            le16_to_cpu(echo),
 910                            le16_to_cpu(p_ent->elem.hdr.echo));
 911         }
 912
 913         /* Release lock before callback, as callback may post
 914          * an additional ramrod.
 915          */
 916         spin_unlock_bh(&p_spq->lock);
 917
 918         if (!found) {
 919                 DP_NOTICE(p_hwfn,
 920                           "Failed to find an entry this EQE [echo %04x] completes\n",
 921                           le16_to_cpu(echo));
 922                 return -EEXIST;
 923         }
 924
 925         DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
 926                    "Complete EQE [echo %04x]: func %p cookie %p)\n",
 927                    le16_to_cpu(echo),
 928                    p_ent->comp_cb.function, p_ent->comp_cb.cookie);
 929         if (found->comp_cb.function)
 930                 found->comp_cb.function(p_hwfn, found->comp_cb.cookie, p_data,
 931                                         fw_return_code);
 932         else
 933                 DP_VERBOSE(p_hwfn,
 934                            QED_MSG_SPQ,
 935                            "Got a completion without a callback function\n");
 936
 937         if ((found->comp_mode != QED_SPQ_MODE_EBLOCK) ||
 938             (found->queue == &p_spq->unlimited_pending))
 939                 /* EBLOCK  is responsible for returning its own entry into the
 940                  * free list, unless it originally added the entry into the
 941                  * unlimited pending list.
 942                  */
 943                 qed_spq_return_entry(p_hwfn, found);
 944
 945         /* Attempt to post pending requests */
 946         spin_lock_bh(&p_spq->lock);
 947         rc = qed_spq_pend_post(p_hwfn);
 948         spin_unlock_bh(&p_spq->lock);
 949
 950         return rc;
 951 }
 952
 953 int qed_consq_alloc(struct qed_hwfn *p_hwfn)
 954 {
 955         struct qed_consq *p_consq;
 956
 957         /* Allocate ConsQ struct */
 958         p_consq = kzalloc(sizeof(*p_consq), GFP_KERNEL);
 959         if (!p_consq)
 960                 return -ENOMEM;
 961
 962         /* Allocate and initialize EQ chain*/
 963         if (qed_chain_alloc(p_hwfn->cdev,
 964                             QED_CHAIN_USE_TO_PRODUCE,
 965                             QED_CHAIN_MODE_PBL,
 966                             QED_CHAIN_CNT_TYPE_U16,
 967                             QED_CHAIN_PAGE_SIZE / 0x80,
 968                             0x80, &p_consq->chain, NULL))
 969                 goto consq_allocate_fail;
 970
 971         p_hwfn->p_consq = p_consq;
 972         return 0;
 973
 974 consq_allocate_fail:
 975         kfree(p_consq);
 976         return -ENOMEM;
 977 }
 978
 979 void qed_consq_setup(struct qed_hwfn *p_hwfn)
 980 {
 981         qed_chain_reset(&p_hwfn->p_consq->chain);
 982 }
 983
 984 void qed_consq_free(struct qed_hwfn *p_hwfn)
 985 {
 986         if (!p_hwfn->p_consq)
 987                 return;
 988
 989         qed_chain_free(p_hwfn->cdev, &p_hwfn->p_consq->chain);
 990
 991         kfree(p_hwfn->p_consq);
 992         p_hwfn->p_consq = NULL;
 993 }