drivers/infiniband/hw/i40iw/i40iw_ctrl.c

   1 /*******************************************************************************
   2 *
   3 * Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.
   4 *
   5 * This software is available to you under a choice of one of two
   6 * licenses.  You may choose to be licensed under the terms of the GNU
   7 * General Public License (GPL) Version 2, available from the file
   8 * COPYING in the main directory of this source tree, or the
   9 * OpenFabrics.org BSD license below:
  10 *
  11 *   Redistribution and use in source and binary forms, with or
  12 *   without modification, are permitted provided that the following
  13 *   conditions are met:
  14 *
  15 *    - Redistributions of source code must retain the above
  16 *       copyright notice, this list of conditions and the following
  17 *       disclaimer.
  18 *
  19 *    - Redistributions in binary form must reproduce the above
  20 *       copyright notice, this list of conditions and the following
  21 *       disclaimer in the documentation and/or other materials
  22 *       provided with the distribution.
  23 *
  24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31 * SOFTWARE.
  32 *
  33 *******************************************************************************/
  34
  35 #include "i40iw_osdep.h"
  36 #include "i40iw_register.h"
  37 #include "i40iw_status.h"
  38 #include "i40iw_hmc.h"
  39
  40 #include "i40iw_d.h"
  41 #include "i40iw_type.h"
  42 #include "i40iw_p.h"
  43 #include "i40iw_vf.h"
  44 #include "i40iw_virtchnl.h"
  45
  46 /**
  47  * i40iw_insert_wqe_hdr - write wqe header
  48  * @wqe: cqp wqe for header
  49  * @header: header for the cqp wqe
  50  */
  51 void i40iw_insert_wqe_hdr(u64 *wqe, u64 header)
  52 {
  53         wmb();            /* make sure WQE is populated before polarity is set */
  54         set_64bit_val(wqe, 24, header);
  55 }
  56
  57 void i40iw_check_cqp_progress(struct i40iw_cqp_timeout *cqp_timeout, struct i40iw_sc_dev *dev)
  58 {
  59         if (cqp_timeout->compl_cqp_cmds != dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]) {
  60                 cqp_timeout->compl_cqp_cmds = dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS];
  61                 cqp_timeout->count = 0;
  62         } else {
  63                 if (dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] != cqp_timeout->compl_cqp_cmds)
  64                         cqp_timeout->count++;
  65         }
  66 }
  67
  68 /**
  69  * i40iw_get_cqp_reg_info - get head and tail for cqp using registers
  70  * @cqp: struct for cqp hw
  71  * @val: cqp tail register value
  72  * @tail:wqtail register value
  73  * @error: cqp processing err
  74  */
  75 static inline void i40iw_get_cqp_reg_info(struct i40iw_sc_cqp *cqp,
  76                                           u32 *val,
  77                                           u32 *tail,
  78                                           u32 *error)
  79 {
  80         if (cqp->dev->is_pf) {
  81                 *val = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPTAIL);
  82                 *tail = RS_32(*val, I40E_PFPE_CQPTAIL_WQTAIL);
  83                 *error = RS_32(*val, I40E_PFPE_CQPTAIL_CQP_OP_ERR);
  84         } else {
  85                 *val = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPTAIL1);
  86                 *tail = RS_32(*val, I40E_VFPE_CQPTAIL_WQTAIL);
  87                 *error = RS_32(*val, I40E_VFPE_CQPTAIL_CQP_OP_ERR);
  88         }
  89 }
  90
  91 /**
  92  * i40iw_cqp_poll_registers - poll cqp registers
  93  * @cqp: struct for cqp hw
  94  * @tail:wqtail register value
  95  * @count: how many times to try for completion
  96  */
  97 static enum i40iw_status_code i40iw_cqp_poll_registers(
  98                                                 struct i40iw_sc_cqp *cqp,
  99                                                 u32 tail,
 100                                                 u32 count)
 101 {
 102         u32 i = 0;
 103         u32 newtail, error, val;
 104
 105         while (i < count) {
 106                 i++;
 107                 i40iw_get_cqp_reg_info(cqp, &val, &newtail, &error);
 108                 if (error) {
 109                         error = (cqp->dev->is_pf) ?
 110                                  i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES) :
 111                                  i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1);
 112                         return I40IW_ERR_CQP_COMPL_ERROR;
 113                 }
 114                 if (newtail != tail) {
 115                         /* SUCCESS */
 116                         I40IW_RING_MOVE_TAIL(cqp->sq_ring);
 117                         cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
 118                         return 0;
 119                 }
 120                 udelay(I40IW_SLEEP_COUNT);
 121         }
 122         return I40IW_ERR_TIMEOUT;
 123 }
 124
 125 /**
 126  * i40iw_sc_parse_fpm_commit_buf - parse fpm commit buffer
 127  * @buf: ptr to fpm commit buffer
 128  * @info: ptr to i40iw_hmc_obj_info struct
 129  * @sd: number of SDs for HMC objects
 130  *
 131  * parses fpm commit info and copy base value
 132  * of hmc objects in hmc_info
 133  */
 134 static enum i40iw_status_code i40iw_sc_parse_fpm_commit_buf(
 135                                 u64 *buf,
 136                                 struct i40iw_hmc_obj_info *info,
 137                                 u32 *sd)
 138 {
 139         u64 temp;
 140         u64 size;
 141         u64 base = 0;
 142         u32 i, j;
 143         u32 k = 0;
 144
 145         /* copy base values in obj_info */
 146         for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE; i++, j += 8) {
 147                 if ((i == I40IW_HMC_IW_SRQ) ||
 148                         (i == I40IW_HMC_IW_FSIMC) ||
 149                         (i == I40IW_HMC_IW_FSIAV)) {
 150                         info[i].base = 0;
 151                         info[i].cnt = 0;
 152                         continue;
 153                 }
 154                 get_64bit_val(buf, j, &temp);
 155                 info[i].base = RS_64_1(temp, 32) * 512;
 156                 if (info[i].base > base) {
 157                         base = info[i].base;
 158                         k = i;
 159                 }
 160                 if (i == I40IW_HMC_IW_APBVT_ENTRY) {
 161                         info[i].cnt = 1;
 162                         continue;
 163                 }
 164                 if (i == I40IW_HMC_IW_QP)
 165                         info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
 166                 else if (i == I40IW_HMC_IW_CQ)
 167                         info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
 168                 else
 169                         info[i].cnt = (u32)(temp);
 170         }
 171         size = info[k].cnt * info[k].size + info[k].base;
 172         if (size & 0x1FFFFF)
 173                 *sd = (u32)((size >> 21) + 1); /* add 1 for remainder */
 174         else
 175                 *sd = (u32)(size >> 21);
 176
 177         return 0;
 178 }
 179
 180 /**
 181  * i40iw_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
 182  * @buf: ptr to fpm query buffer
 183  * @buf_idx: index into buf
 184  * @info: ptr to i40iw_hmc_obj_info struct
 185  * @rsrc_idx: resource index into info
 186  *
 187  * Decode a 64 bit value from fpm query buffer into max count and size
 188  */
 189 static u64 i40iw_sc_decode_fpm_query(u64 *buf,
 190                                             u32 buf_idx,
 191                                             struct i40iw_hmc_obj_info *obj_info,
 192                                             u32 rsrc_idx)
 193 {
 194         u64 temp;
 195         u32 size;
 196
 197         get_64bit_val(buf, buf_idx, &temp);
 198         obj_info[rsrc_idx].max_cnt = (u32)temp;
 199         size = (u32)RS_64_1(temp, 32);
 200         obj_info[rsrc_idx].size = LS_64_1(1, size);
 201
 202         return temp;
 203 }
 204
 205 /**
 206  * i40iw_sc_parse_fpm_query_buf() - parses fpm query buffer
 207  * @buf: ptr to fpm query buffer
 208  * @info: ptr to i40iw_hmc_obj_info struct
 209  * @hmc_fpm_misc: ptr to fpm data
 210  *
 211  * parses fpm query buffer and copy max_cnt and
 212  * size value of hmc objects in hmc_info
 213  */
 214 static enum i40iw_status_code i40iw_sc_parse_fpm_query_buf(
 215                                 u64 *buf,
 216                                 struct i40iw_hmc_info *hmc_info,
 217                                 struct i40iw_hmc_fpm_misc *hmc_fpm_misc)
 218 {
 219         struct i40iw_hmc_obj_info *obj_info;
 220         u64 temp;
 221         u32 size;
 222         u16 max_pe_sds;
 223
 224         obj_info = hmc_info->hmc_obj;
 225
 226         get_64bit_val(buf, 0, &temp);
 227         hmc_info->first_sd_index = (u16)RS_64(temp, I40IW_QUERY_FPM_FIRST_PE_SD_INDEX);
 228         max_pe_sds = (u16)RS_64(temp, I40IW_QUERY_FPM_MAX_PE_SDS);
 229
 230         /* Reduce SD count for VFs by 1 to account for PBLE backing page rounding */
 231         if (hmc_info->hmc_fn_id >= I40IW_FIRST_VF_FPM_ID)
 232                 max_pe_sds--;
 233         hmc_fpm_misc->max_sds = max_pe_sds;
 234         hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;
 235
 236         get_64bit_val(buf, 8, &temp);
 237         obj_info[I40IW_HMC_IW_QP].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
 238         size = (u32)RS_64_1(temp, 32);
 239         obj_info[I40IW_HMC_IW_QP].size = LS_64_1(1, size);
 240
 241         get_64bit_val(buf, 16, &temp);
 242         obj_info[I40IW_HMC_IW_CQ].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
 243         size = (u32)RS_64_1(temp, 32);
 244         obj_info[I40IW_HMC_IW_CQ].size = LS_64_1(1, size);
 245
 246         i40iw_sc_decode_fpm_query(buf, 32, obj_info, I40IW_HMC_IW_HTE);
 247         i40iw_sc_decode_fpm_query(buf, 40, obj_info, I40IW_HMC_IW_ARP);
 248
 249         obj_info[I40IW_HMC_IW_APBVT_ENTRY].size = 8192;
 250         obj_info[I40IW_HMC_IW_APBVT_ENTRY].max_cnt = 1;
 251
 252         i40iw_sc_decode_fpm_query(buf, 48, obj_info, I40IW_HMC_IW_MR);
 253         i40iw_sc_decode_fpm_query(buf, 56, obj_info, I40IW_HMC_IW_XF);
 254
 255         get_64bit_val(buf, 64, &temp);
 256         obj_info[I40IW_HMC_IW_XFFL].max_cnt = (u32)temp;
 257         obj_info[I40IW_HMC_IW_XFFL].size = 4;
 258         hmc_fpm_misc->xf_block_size = RS_64(temp, I40IW_QUERY_FPM_XFBLOCKSIZE);
 259         if (!hmc_fpm_misc->xf_block_size)
 260                 return I40IW_ERR_INVALID_SIZE;
 261
 262         i40iw_sc_decode_fpm_query(buf, 72, obj_info, I40IW_HMC_IW_Q1);
 263
 264         get_64bit_val(buf, 80, &temp);
 265         obj_info[I40IW_HMC_IW_Q1FL].max_cnt = (u32)temp;
 266         obj_info[I40IW_HMC_IW_Q1FL].size = 4;
 267         hmc_fpm_misc->q1_block_size = RS_64(temp, I40IW_QUERY_FPM_Q1BLOCKSIZE);
 268         if (!hmc_fpm_misc->q1_block_size)
 269                 return I40IW_ERR_INVALID_SIZE;
 270
 271         i40iw_sc_decode_fpm_query(buf, 88, obj_info, I40IW_HMC_IW_TIMER);
 272
 273         get_64bit_val(buf, 112, &temp);
 274         obj_info[I40IW_HMC_IW_PBLE].max_cnt = (u32)temp;
 275         obj_info[I40IW_HMC_IW_PBLE].size = 8;
 276
 277         get_64bit_val(buf, 120, &temp);
 278         hmc_fpm_misc->max_ceqs = (u8)RS_64(temp, I40IW_QUERY_FPM_MAX_CEQS);
 279         hmc_fpm_misc->ht_multiplier = RS_64(temp, I40IW_QUERY_FPM_HTMULTIPLIER);
 280         hmc_fpm_misc->timer_bucket = RS_64(temp, I40IW_QUERY_FPM_TIMERBUCKET);
 281
 282         return 0;
 283 }
 284
 285 /**
 286  * i40iw_fill_qos_list - Change all unknown qs handles to available ones
 287  * @qs_list: list of qs_handles to be fixed with valid qs_handles
 288  */
 289 static void i40iw_fill_qos_list(u16 *qs_list)
 290 {
 291         u16 qshandle = qs_list[0];
 292         int i;
 293
 294         for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
 295                 if (qs_list[i] == QS_HANDLE_UNKNOWN)
 296                         qs_list[i] = qshandle;
 297                 else
 298                         qshandle = qs_list[i];
 299         }
 300 }
 301
 302 /**
 303  * i40iw_qp_from_entry - Given entry, get to the qp structure
 304  * @entry: Points to list of qp structure
 305  */
 306 static struct i40iw_sc_qp *i40iw_qp_from_entry(struct list_head *entry)
 307 {
 308         if (!entry)
 309                 return NULL;
 310
 311         return (struct i40iw_sc_qp *)((char *)entry - offsetof(struct i40iw_sc_qp, list));
 312 }
 313
 314 /**
 315  * i40iw_get_qp - get the next qp from the list given current qp
 316  * @head: Listhead of qp's
 317  * @qp: current qp
 318  */
 319 static struct i40iw_sc_qp *i40iw_get_qp(struct list_head *head, struct i40iw_sc_qp *qp)
 320 {
 321         struct list_head *entry = NULL;
 322         struct list_head *lastentry;
 323
 324         if (list_empty(head))
 325                 return NULL;
 326
 327         if (!qp) {
 328                 entry = head->next;
 329         } else {
 330                 lastentry = &qp->list;
 331                 entry = (lastentry != head) ? lastentry->next : NULL;
 332         }
 333
 334         return i40iw_qp_from_entry(entry);
 335 }
 336
 337 /**
 338  * i40iw_change_l2params - given the new l2 parameters, change all qp
 339  * @vsi: pointer to the vsi structure
 340  * @l2params: New paramaters from l2
 341  */
 342 void i40iw_change_l2params(struct i40iw_sc_vsi *vsi, struct i40iw_l2params *l2params)
 343 {
 344         struct i40iw_sc_dev *dev = vsi->dev;
 345         struct i40iw_sc_qp *qp = NULL;
 346         bool qs_handle_change = false;
 347         unsigned long flags;
 348         u16 qs_handle;
 349         int i;
 350
 351         if (vsi->mtu != l2params->mtu) {
 352                 vsi->mtu = l2params->mtu;
 353                 i40iw_reinitialize_ieq(dev);
 354         }
 355
 356         i40iw_fill_qos_list(l2params->qs_handle_list);
 357         for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
 358                 qs_handle = l2params->qs_handle_list[i];
 359                 if (vsi->qos[i].qs_handle != qs_handle)
 360                         qs_handle_change = true;
 361                 spin_lock_irqsave(&vsi->qos[i].lock, flags);
 362                 qp = i40iw_get_qp(&vsi->qos[i].qplist, qp);
 363                 while (qp) {
 364                         if (qs_handle_change) {
 365                                 qp->qs_handle = qs_handle;
 366                                 /* issue cqp suspend command */
 367                                 i40iw_qp_suspend_resume(dev, qp, true);
 368                         }
 369                         qp = i40iw_get_qp(&vsi->qos[i].qplist, qp);
 370                 }
 371                 spin_unlock_irqrestore(&vsi->qos[i].lock, flags);
 372                 vsi->qos[i].qs_handle = qs_handle;
 373         }
 374 }
 375
 376 /**
 377  * i40iw_qp_rem_qos - remove qp from qos lists during destroy qp
 378  * @qp: qp to be removed from qos
 379  */
 380 void i40iw_qp_rem_qos(struct i40iw_sc_qp *qp)
 381 {
 382         struct i40iw_sc_vsi *vsi = qp->vsi;
 383         unsigned long flags;
 384
 385         if (!qp->on_qoslist)
 386                 return;
 387         spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
 388         list_del(&qp->list);
 389         spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
 390 }
 391
 392 /**
 393  * i40iw_qp_add_qos - called during setctx fot qp to be added to qos
 394  * @qp: qp to be added to qos
 395  */
 396 void i40iw_qp_add_qos(struct i40iw_sc_qp *qp)
 397 {
 398         struct i40iw_sc_vsi *vsi = qp->vsi;
 399         unsigned long flags;
 400
 401         if (qp->on_qoslist)
 402                 return;
 403         spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
 404         qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;
 405         list_add(&qp->list, &vsi->qos[qp->user_pri].qplist);
 406         qp->on_qoslist = true;
 407         spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
 408 }
 409
 410 /**
 411  * i40iw_sc_pd_init - initialize sc pd struct
 412  * @dev: sc device struct
 413  * @pd: sc pd ptr
 414  * @pd_id: pd_id for allocated pd
 415  * @abi_ver: ABI version from user context, -1 if not valid
 416  */
 417 static void i40iw_sc_pd_init(struct i40iw_sc_dev *dev,
 418                              struct i40iw_sc_pd *pd,
 419                              u16 pd_id,
 420                              int abi_ver)
 421 {
 422         pd->size = sizeof(*pd);
 423         pd->pd_id = pd_id;
 424         pd->abi_ver = abi_ver;
 425         pd->dev = dev;
 426 }
 427
 428 /**
 429  * i40iw_get_encoded_wqe_size - given wq size, returns hardware encoded size
 430  * @wqsize: size of the wq (sq, rq, srq) to encoded_size
 431  * @cqpsq: encoded size for sq for cqp as its encoded size is 1+ other wq's
 432  */
 433 u8 i40iw_get_encoded_wqe_size(u32 wqsize, bool cqpsq)
 434 {
 435         u8 encoded_size = 0;
 436
 437         /* cqp sq's hw coded value starts from 1 for size of 4
 438          * while it starts from 0 for qp' wq's.
 439          */
 440         if (cqpsq)
 441                 encoded_size = 1;
 442         wqsize >>= 2;
 443         while (wqsize >>= 1)
 444                 encoded_size++;
 445         return encoded_size;
 446 }
 447
 448 /**
 449  * i40iw_sc_cqp_init - Initialize buffers for a control Queue Pair
 450  * @cqp: IWARP control queue pair pointer
 451  * @info: IWARP control queue pair init info pointer
 452  *
 453  * Initializes the object and context buffers for a control Queue Pair.
 454  */
 455 static enum i40iw_status_code i40iw_sc_cqp_init(struct i40iw_sc_cqp *cqp,
 456                                                 struct i40iw_cqp_init_info *info)
 457 {
 458         u8 hw_sq_size;
 459
 460         if ((info->sq_size > I40IW_CQP_SW_SQSIZE_2048) ||
 461             (info->sq_size < I40IW_CQP_SW_SQSIZE_4) ||
 462             ((info->sq_size & (info->sq_size - 1))))
 463                 return I40IW_ERR_INVALID_SIZE;
 464
 465         hw_sq_size = i40iw_get_encoded_wqe_size(info->sq_size, true);
 466         cqp->size = sizeof(*cqp);
 467         cqp->sq_size = info->sq_size;
 468         cqp->hw_sq_size = hw_sq_size;
 469         cqp->sq_base = info->sq;
 470         cqp->host_ctx = info->host_ctx;
 471         cqp->sq_pa = info->sq_pa;
 472         cqp->host_ctx_pa = info->host_ctx_pa;
 473         cqp->dev = info->dev;
 474         cqp->struct_ver = info->struct_ver;
 475         cqp->scratch_array = info->scratch_array;
 476         cqp->polarity = 0;
 477         cqp->en_datacenter_tcp = info->en_datacenter_tcp;
 478         cqp->enabled_vf_count = info->enabled_vf_count;
 479         cqp->hmc_profile = info->hmc_profile;
 480         info->dev->cqp = cqp;
 481
 482         I40IW_RING_INIT(cqp->sq_ring, cqp->sq_size);
 483         cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] = 0;
 484         cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS] = 0;
 485         INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head);               /* for the cqp commands backlog. */
 486
 487         i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPTAIL, 0);
 488         i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPDB, 0);
 489
 490         i40iw_debug(cqp->dev, I40IW_DEBUG_WQE,
 491                     "%s: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04X]\n",
 492                     __func__, cqp->sq_size, cqp->hw_sq_size,
 493                     cqp->sq_base, cqp->sq_pa, cqp, cqp->polarity);
 494         return 0;
 495 }
 496
 497 /**
 498  * i40iw_sc_cqp_create - create cqp during bringup
 499  * @cqp: struct for cqp hw
 500  * @maj_err: If error, major err number
 501  * @min_err: If error, minor err number
 502  */
 503 static enum i40iw_status_code i40iw_sc_cqp_create(struct i40iw_sc_cqp *cqp,
 504                                                   u16 *maj_err,
 505                                                   u16 *min_err)
 506 {
 507         u64 temp;
 508         u32 cnt = 0, p1, p2, val = 0, err_code;
 509         enum i40iw_status_code ret_code;
 510
 511         *maj_err = 0;
 512         *min_err = 0;
 513
 514         ret_code = i40iw_allocate_dma_mem(cqp->dev->hw,
 515                                           &cqp->sdbuf,
 516                                           I40IW_UPDATE_SD_BUF_SIZE * cqp->sq_size,
 517                                           I40IW_SD_BUF_ALIGNMENT);
 518
 519         if (ret_code)
 520                 goto exit;
 521
 522         temp = LS_64(cqp->hw_sq_size, I40IW_CQPHC_SQSIZE) |
 523                LS_64(cqp->struct_ver, I40IW_CQPHC_SVER);
 524
 525         set_64bit_val(cqp->host_ctx, 0, temp);
 526         set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa);
 527         temp = LS_64(cqp->enabled_vf_count, I40IW_CQPHC_ENABLED_VFS) |
 528                LS_64(cqp->hmc_profile, I40IW_CQPHC_HMC_PROFILE);
 529         set_64bit_val(cqp->host_ctx, 16, temp);
 530         set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp);
 531         set_64bit_val(cqp->host_ctx, 32, 0);
 532         set_64bit_val(cqp->host_ctx, 40, 0);
 533         set_64bit_val(cqp->host_ctx, 48, 0);
 534         set_64bit_val(cqp->host_ctx, 56, 0);
 535
 536         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQP_HOST_CTX",
 537                         cqp->host_ctx, I40IW_CQP_CTX_SIZE * 8);
 538
 539         p1 = RS_32_1(cqp->host_ctx_pa, 32);
 540         p2 = (u32)cqp->host_ctx_pa;
 541
 542         if (cqp->dev->is_pf) {
 543                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, p1);
 544                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, p2);
 545         } else {
 546                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, p1);
 547                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, p2);
 548         }
 549         do {
 550                 if (cnt++ > I40IW_DONE_COUNT) {
 551                         i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
 552                         ret_code = I40IW_ERR_TIMEOUT;
 553                         /*
 554                          * read PFPE_CQPERRORCODES register to get the minor
 555                          * and major error code
 556                          */
 557                         if (cqp->dev->is_pf)
 558                                 err_code = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES);
 559                         else
 560                                 err_code = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1);
 561                         *min_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE);
 562                         *maj_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE);
 563                         goto exit;
 564                 }
 565                 udelay(I40IW_SLEEP_COUNT);
 566                 if (cqp->dev->is_pf)
 567                         val = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CCQPSTATUS);
 568                 else
 569                         val = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CCQPSTATUS1);
 570         } while (!val);
 571
 572 exit:
 573         if (!ret_code)
 574                 cqp->process_cqp_sds = i40iw_update_sds_noccq;
 575         return ret_code;
 576 }
 577
 578 /**
 579  * i40iw_sc_cqp_post_sq - post of cqp's sq
 580  * @cqp: struct for cqp hw
 581  */
 582 void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp)
 583 {
 584         if (cqp->dev->is_pf)
 585                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPDB, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring));
 586         else
 587                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CQPDB1, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring));
 588
 589         i40iw_debug(cqp->dev,
 590                     I40IW_DEBUG_WQE,
 591                     "%s: HEAD_TAIL[%04d,%04d,%04d]\n",
 592                     __func__,
 593                     cqp->sq_ring.head,
 594                     cqp->sq_ring.tail,
 595                     cqp->sq_ring.size);
 596 }
 597
 598 /**
 599  * i40iw_sc_cqp_get_next_send_wqe_idx - get next WQE on CQP SQ and pass back the index
 600  * @cqp: pointer to CQP structure
 601  * @scratch: private data for CQP WQE
 602  * @wqe_idx: WQE index for next WQE on CQP SQ
 603  */
 604 static u64 *i40iw_sc_cqp_get_next_send_wqe_idx(struct i40iw_sc_cqp *cqp,
 605                                                u64 scratch, u32 *wqe_idx)
 606 {
 607         u64 *wqe = NULL;
 608         enum i40iw_status_code ret_code;
 609
 610         if (I40IW_RING_FULL_ERR(cqp->sq_ring)) {
 611                 i40iw_debug(cqp->dev,
 612                             I40IW_DEBUG_WQE,
 613                             "%s: ring is full head %x tail %x size %x\n",
 614                             __func__,
 615                             cqp->sq_ring.head,
 616                             cqp->sq_ring.tail,
 617                             cqp->sq_ring.size);
 618                 return NULL;
 619         }
 620         I40IW_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code);
 621         cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS]++;
 622         if (ret_code)
 623                 return NULL;
 624         if (!*wqe_idx)
 625                 cqp->polarity = !cqp->polarity;
 626
 627         wqe = cqp->sq_base[*wqe_idx].elem;
 628         cqp->scratch_array[*wqe_idx] = scratch;
 629         I40IW_CQP_INIT_WQE(wqe);
 630
 631         return wqe;
 632 }
 633
 634 /**
 635  * i40iw_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
 636  * @cqp: struct for cqp hw
 637  * @scratch: private data for CQP WQE
 638  */
 639 u64 *i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch)
 640 {
 641         u32 wqe_idx;
 642
 643         return i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
 644 }
 645
 646 /**
 647  * i40iw_sc_cqp_destroy - destroy cqp during close
 648  * @cqp: struct for cqp hw
 649  */
 650 static enum i40iw_status_code i40iw_sc_cqp_destroy(struct i40iw_sc_cqp *cqp)
 651 {
 652         u32 cnt = 0, val = 1;
 653         enum i40iw_status_code ret_code = 0;
 654         u32 cqpstat_addr;
 655
 656         if (cqp->dev->is_pf) {
 657                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, 0);
 658                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, 0);
 659                 cqpstat_addr = I40E_PFPE_CCQPSTATUS;
 660         } else {
 661                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, 0);
 662                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, 0);
 663                 cqpstat_addr = I40E_VFPE_CCQPSTATUS1;
 664         }
 665         do {
 666                 if (cnt++ > I40IW_DONE_COUNT) {
 667                         ret_code = I40IW_ERR_TIMEOUT;
 668                         break;
 669                 }
 670                 udelay(I40IW_SLEEP_COUNT);
 671                 val = i40iw_rd32(cqp->dev->hw, cqpstat_addr);
 672         } while (val);
 673
 674         i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
 675         return ret_code;
 676 }
 677
 678 /**
 679  * i40iw_sc_ccq_arm - enable intr for control cq
 680  * @ccq: ccq sc struct
 681  */
 682 static void i40iw_sc_ccq_arm(struct i40iw_sc_cq *ccq)
 683 {
 684         u64 temp_val;
 685         u16 sw_cq_sel;
 686         u8 arm_next_se;
 687         u8 arm_seq_num;
 688
 689         /* write to cq doorbell shadow area */
 690         /* arm next se should always be zero */
 691         get_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val);
 692
 693         sw_cq_sel = (u16)RS_64(temp_val, I40IW_CQ_DBSA_SW_CQ_SELECT);
 694         arm_next_se = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_NEXT_SE);
 695
 696         arm_seq_num = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_SEQ_NUM);
 697         arm_seq_num++;
 698
 699         temp_val = LS_64(arm_seq_num, I40IW_CQ_DBSA_ARM_SEQ_NUM) |
 700                    LS_64(sw_cq_sel, I40IW_CQ_DBSA_SW_CQ_SELECT) |
 701                    LS_64(arm_next_se, I40IW_CQ_DBSA_ARM_NEXT_SE) |
 702                    LS_64(1, I40IW_CQ_DBSA_ARM_NEXT);
 703
 704         set_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val);
 705
 706         wmb();       /* make sure shadow area is updated before arming */
 707
 708         if (ccq->dev->is_pf)
 709                 i40iw_wr32(ccq->dev->hw, I40E_PFPE_CQARM, ccq->cq_uk.cq_id);
 710         else
 711                 i40iw_wr32(ccq->dev->hw, I40E_VFPE_CQARM1, ccq->cq_uk.cq_id);
 712 }
 713
 714 /**
 715  * i40iw_sc_ccq_get_cqe_info - get ccq's cq entry
 716  * @ccq: ccq sc struct
 717  * @info: completion q entry to return
 718  */
 719 static enum i40iw_status_code i40iw_sc_ccq_get_cqe_info(
 720                                         struct i40iw_sc_cq *ccq,
 721                                         struct i40iw_ccq_cqe_info *info)
 722 {
 723         u64 qp_ctx, temp, temp1;
 724         u64 *cqe;
 725         struct i40iw_sc_cqp *cqp;
 726         u32 wqe_idx;
 727         u8 polarity;
 728         enum i40iw_status_code ret_code = 0;
 729
 730         if (ccq->cq_uk.avoid_mem_cflct)
 731                 cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(&ccq->cq_uk);
 732         else
 733                 cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(&ccq->cq_uk);
 734
 735         get_64bit_val(cqe, 24, &temp);
 736         polarity = (u8)RS_64(temp, I40IW_CQ_VALID);
 737         if (polarity != ccq->cq_uk.polarity)
 738                 return I40IW_ERR_QUEUE_EMPTY;
 739
 740         get_64bit_val(cqe, 8, &qp_ctx);
 741         cqp = (struct i40iw_sc_cqp *)(unsigned long)qp_ctx;
 742         info->error = (bool)RS_64(temp, I40IW_CQ_ERROR);
 743         info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);
 744         if (info->error) {
 745                 info->maj_err_code = (u16)RS_64(temp, I40IW_CQ_MAJERR);
 746                 info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);
 747         }
 748         wqe_idx = (u32)RS_64(temp, I40IW_CQ_WQEIDX);
 749         info->scratch = cqp->scratch_array[wqe_idx];
 750
 751         get_64bit_val(cqe, 16, &temp1);
 752         info->op_ret_val = (u32)RS_64(temp1, I40IW_CCQ_OPRETVAL);
 753         get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1);
 754         info->op_code = (u8)RS_64(temp1, I40IW_CQPSQ_OPCODE);
 755         info->cqp = cqp;
 756
 757         /*  move the head for cq */
 758         I40IW_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code);
 759         if (I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring) == 0)
 760                 ccq->cq_uk.polarity ^= 1;
 761
 762         /* update cq tail in cq shadow memory also */
 763         I40IW_RING_MOVE_TAIL(ccq->cq_uk.cq_ring);
 764         set_64bit_val(ccq->cq_uk.shadow_area,
 765                       0,
 766                       I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring));
 767         wmb(); /* write shadow area before tail */
 768         I40IW_RING_MOVE_TAIL(cqp->sq_ring);
 769         ccq->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
 770
 771         return ret_code;
 772 }
 773
 774 /**
 775  * i40iw_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
 776  * @cqp: struct for cqp hw
 777  * @op_code: cqp opcode for completion
 778  * @info: completion q entry to return
 779  */
 780 static enum i40iw_status_code i40iw_sc_poll_for_cqp_op_done(
 781                                         struct i40iw_sc_cqp *cqp,
 782                                         u8 op_code,
 783                                         struct i40iw_ccq_cqe_info *compl_info)
 784 {
 785         struct i40iw_ccq_cqe_info info;
 786         struct i40iw_sc_cq *ccq;
 787         enum i40iw_status_code ret_code = 0;
 788         u32 cnt = 0;
 789
 790         memset(&info, 0, sizeof(info));
 791         ccq = cqp->dev->ccq;
 792         while (1) {
 793                 if (cnt++ > I40IW_DONE_COUNT)
 794                         return I40IW_ERR_TIMEOUT;
 795
 796                 if (i40iw_sc_ccq_get_cqe_info(ccq, &info)) {
 797                         udelay(I40IW_SLEEP_COUNT);
 798                         continue;
 799                 }
 800
 801                 if (info.error) {
 802                         ret_code = I40IW_ERR_CQP_COMPL_ERROR;
 803                         break;
 804                 }
 805                 /* check if opcode is cq create */
 806                 if (op_code != info.op_code) {
 807                         i40iw_debug(cqp->dev, I40IW_DEBUG_WQE,
 808                                     "%s: opcode mismatch for my op code 0x%x, returned opcode %x\n",
 809                                     __func__, op_code, info.op_code);
 810                 }
 811                 /* success, exit out of the loop */
 812                 if (op_code == info.op_code)
 813                         break;
 814         }
 815
 816         if (compl_info)
 817                 memcpy(compl_info, &info, sizeof(*compl_info));
 818
 819         return ret_code;
 820 }
 821
 822 /**
 823  * i40iw_sc_manage_push_page - Handle push page
 824  * @cqp: struct for cqp hw
 825  * @info: push page info
 826  * @scratch: u64 saved to be used during cqp completion
 827  * @post_sq: flag for cqp db to ring
 828  */
 829 static enum i40iw_status_code i40iw_sc_manage_push_page(
 830                                 struct i40iw_sc_cqp *cqp,
 831                                 struct i40iw_cqp_manage_push_page_info *info,
 832                                 u64 scratch,
 833                                 bool post_sq)
 834 {
 835         u64 *wqe;
 836         u64 header;
 837
 838         if (info->push_idx >= I40IW_MAX_PUSH_PAGE_COUNT)
 839                 return I40IW_ERR_INVALID_PUSH_PAGE_INDEX;
 840
 841         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 842         if (!wqe)
 843                 return I40IW_ERR_RING_FULL;
 844
 845         set_64bit_val(wqe, 16, info->qs_handle);
 846
 847         header = LS_64(info->push_idx, I40IW_CQPSQ_MPP_PPIDX) |
 848                  LS_64(I40IW_CQP_OP_MANAGE_PUSH_PAGES, I40IW_CQPSQ_OPCODE) |
 849                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
 850                  LS_64(info->free_page, I40IW_CQPSQ_MPP_FREE_PAGE);
 851
 852         i40iw_insert_wqe_hdr(wqe, header);
 853
 854         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_PUSH_PAGES WQE",
 855                         wqe, I40IW_CQP_WQE_SIZE * 8);
 856
 857         if (post_sq)
 858                 i40iw_sc_cqp_post_sq(cqp);
 859         return 0;
 860 }
 861
 862 /**
 863  * i40iw_sc_manage_hmc_pm_func_table - manage of function table
 864  * @cqp: struct for cqp hw
 865  * @scratch: u64 saved to be used during cqp completion
 866  * @vf_index: vf index for cqp
 867  * @free_pm_fcn: function number
 868  * @post_sq: flag for cqp db to ring
 869  */
 870 static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table(
 871                                 struct i40iw_sc_cqp *cqp,
 872                                 u64 scratch,
 873                                 u8 vf_index,
 874                                 bool free_pm_fcn,
 875                                 bool post_sq)
 876 {
 877         u64 *wqe;
 878         u64 header;
 879
 880         if (vf_index >= I40IW_MAX_VF_PER_PF)
 881                 return I40IW_ERR_INVALID_VF_ID;
 882         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 883         if (!wqe)
 884                 return I40IW_ERR_RING_FULL;
 885
 886         header = LS_64(vf_index, I40IW_CQPSQ_MHMC_VFIDX) |
 887                  LS_64(I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, I40IW_CQPSQ_OPCODE) |
 888                  LS_64(free_pm_fcn, I40IW_CQPSQ_MHMC_FREEPMFN) |
 889                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
 890
 891         i40iw_insert_wqe_hdr(wqe, header);
 892         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",
 893                         wqe, I40IW_CQP_WQE_SIZE * 8);
 894         if (post_sq)
 895                 i40iw_sc_cqp_post_sq(cqp);
 896         return 0;
 897 }
 898
 899 /**
 900  * i40iw_sc_set_hmc_resource_profile - cqp wqe for hmc profile
 901  * @cqp: struct for cqp hw
 902  * @scratch: u64 saved to be used during cqp completion
 903  * @hmc_profile_type: type of profile to set
 904  * @vf_num: vf number for profile
 905  * @post_sq: flag for cqp db to ring
 906  * @poll_registers: flag to poll register for cqp completion
 907  */
 908 static enum i40iw_status_code i40iw_sc_set_hmc_resource_profile(
 909                                 struct i40iw_sc_cqp *cqp,
 910                                 u64 scratch,
 911                                 u8 hmc_profile_type,
 912                                 u8 vf_num, bool post_sq,
 913                                 bool poll_registers)
 914 {
 915         u64 *wqe;
 916         u64 header;
 917         u32 val, tail, error;
 918         enum i40iw_status_code ret_code = 0;
 919
 920         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 921         if (!wqe)
 922                 return I40IW_ERR_RING_FULL;
 923
 924         set_64bit_val(wqe, 16,
 925                       (LS_64(hmc_profile_type, I40IW_CQPSQ_SHMCRP_HMC_PROFILE) |
 926                                 LS_64(vf_num, I40IW_CQPSQ_SHMCRP_VFNUM)));
 927
 928         header = LS_64(I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE, I40IW_CQPSQ_OPCODE) |
 929                        LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
 930
 931         i40iw_insert_wqe_hdr(wqe, header);
 932
 933         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",
 934                         wqe, I40IW_CQP_WQE_SIZE * 8);
 935
 936         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
 937         if (error)
 938                 return I40IW_ERR_CQP_COMPL_ERROR;
 939
 940         if (post_sq) {
 941                 i40iw_sc_cqp_post_sq(cqp);
 942                 if (poll_registers)
 943                         ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000000);
 944                 else
 945                         ret_code = i40iw_sc_poll_for_cqp_op_done(cqp,
 946                                                                  I40IW_CQP_OP_SHMC_PAGES_ALLOCATED,
 947                                                                  NULL);
 948         }
 949
 950         return ret_code;
 951 }
 952
 953 /**
 954  * i40iw_sc_manage_hmc_pm_func_table_done - wait for cqp wqe completion for function table
 955  * @cqp: struct for cqp hw
 956  */
 957 static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table_done(struct i40iw_sc_cqp *cqp)
 958 {
 959         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, NULL);
 960 }
 961
 962 /**
 963  * i40iw_sc_commit_fpm_values_done - wait for cqp eqe completion for fpm commit
 964  * @cqp: struct for cqp hw
 965  */
 966 static enum i40iw_status_code i40iw_sc_commit_fpm_values_done(struct i40iw_sc_cqp *cqp)
 967 {
 968         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_COMMIT_FPM_VALUES, NULL);
 969 }
 970
 971 /**
 972  * i40iw_sc_commit_fpm_values - cqp wqe for commit fpm values
 973  * @cqp: struct for cqp hw
 974  * @scratch: u64 saved to be used during cqp completion
 975  * @hmc_fn_id: hmc function id
 976  * @commit_fpm_mem; Memory for fpm values
 977  * @post_sq: flag for cqp db to ring
 978  * @wait_type: poll ccq or cqp registers for cqp completion
 979  */
 980 static enum i40iw_status_code i40iw_sc_commit_fpm_values(
 981                                         struct i40iw_sc_cqp *cqp,
 982                                         u64 scratch,
 983                                         u8 hmc_fn_id,
 984                                         struct i40iw_dma_mem *commit_fpm_mem,
 985                                         bool post_sq,
 986                                         u8 wait_type)
 987 {
 988         u64 *wqe;
 989         u64 header;
 990         u32 tail, val, error;
 991         enum i40iw_status_code ret_code = 0;
 992
 993         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 994         if (!wqe)
 995                 return I40IW_ERR_RING_FULL;
 996
 997         set_64bit_val(wqe, 16, hmc_fn_id);
 998         set_64bit_val(wqe, 32, commit_fpm_mem->pa);
 999
1000         header = LS_64(I40IW_CQP_OP_COMMIT_FPM_VALUES, I40IW_CQPSQ_OPCODE) |
1001                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1002
1003         i40iw_insert_wqe_hdr(wqe, header);
1004
1005         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "COMMIT_FPM_VALUES WQE",
1006                         wqe, I40IW_CQP_WQE_SIZE * 8);
1007
1008         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
1009         if (error)
1010                 return I40IW_ERR_CQP_COMPL_ERROR;
1011
1012         if (post_sq) {
1013                 i40iw_sc_cqp_post_sq(cqp);
1014
1015                 if (wait_type == I40IW_CQP_WAIT_POLL_REGS)
1016                         ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
1017                 else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)
1018                         ret_code = i40iw_sc_commit_fpm_values_done(cqp);
1019         }
1020
1021         return ret_code;
1022 }
1023
1024 /**
1025  * i40iw_sc_query_fpm_values_done - poll for cqp wqe completion for query fpm
1026  * @cqp: struct for cqp hw
1027  */
1028 static enum i40iw_status_code i40iw_sc_query_fpm_values_done(struct i40iw_sc_cqp *cqp)
1029 {
1030         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_QUERY_FPM_VALUES, NULL);
1031 }
1032
1033 /**
1034  * i40iw_sc_query_fpm_values - cqp wqe query fpm values
1035  * @cqp: struct for cqp hw
1036  * @scratch: u64 saved to be used during cqp completion
1037  * @hmc_fn_id: hmc function id
1038  * @query_fpm_mem: memory for return fpm values
1039  * @post_sq: flag for cqp db to ring
1040  * @wait_type: poll ccq or cqp registers for cqp completion
1041  */
1042 static enum i40iw_status_code i40iw_sc_query_fpm_values(
1043                                         struct i40iw_sc_cqp *cqp,
1044                                         u64 scratch,
1045                                         u8 hmc_fn_id,
1046                                         struct i40iw_dma_mem *query_fpm_mem,
1047                                         bool post_sq,
1048                                         u8 wait_type)
1049 {
1050         u64 *wqe;
1051         u64 header;
1052         u32 tail, val, error;
1053         enum i40iw_status_code ret_code = 0;
1054
1055         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1056         if (!wqe)
1057                 return I40IW_ERR_RING_FULL;
1058
1059         set_64bit_val(wqe, 16, hmc_fn_id);
1060         set_64bit_val(wqe, 32, query_fpm_mem->pa);
1061
1062         header = LS_64(I40IW_CQP_OP_QUERY_FPM_VALUES, I40IW_CQPSQ_OPCODE) |
1063                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1064
1065         i40iw_insert_wqe_hdr(wqe, header);
1066
1067         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_FPM WQE",
1068                         wqe, I40IW_CQP_WQE_SIZE * 8);
1069
1070         /* read the tail from CQP_TAIL register */
1071         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
1072
1073         if (error)
1074                 return I40IW_ERR_CQP_COMPL_ERROR;
1075
1076         if (post_sq) {
1077                 i40iw_sc_cqp_post_sq(cqp);
1078                 if (wait_type == I40IW_CQP_WAIT_POLL_REGS)
1079                         ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
1080                 else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)
1081                         ret_code = i40iw_sc_query_fpm_values_done(cqp);
1082         }
1083
1084         return ret_code;
1085 }
1086
1087 /**
1088  * i40iw_sc_add_arp_cache_entry - cqp wqe add arp cache entry
1089  * @cqp: struct for cqp hw
1090  * @info: arp entry information
1091  * @scratch: u64 saved to be used during cqp completion
1092  * @post_sq: flag for cqp db to ring
1093  */
1094 static enum i40iw_status_code i40iw_sc_add_arp_cache_entry(
1095                                 struct i40iw_sc_cqp *cqp,
1096                                 struct i40iw_add_arp_cache_entry_info *info,
1097                                 u64 scratch,
1098                                 bool post_sq)
1099 {
1100         u64 *wqe;
1101         u64 temp, header;
1102
1103         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1104         if (!wqe)
1105                 return I40IW_ERR_RING_FULL;
1106         set_64bit_val(wqe, 8, info->reach_max);
1107
1108         temp = info->mac_addr[5] |
1109                LS_64_1(info->mac_addr[4], 8) |
1110                LS_64_1(info->mac_addr[3], 16) |
1111                LS_64_1(info->mac_addr[2], 24) |
1112                LS_64_1(info->mac_addr[1], 32) |
1113                LS_64_1(info->mac_addr[0], 40);
1114
1115         set_64bit_val(wqe, 16, temp);
1116
1117         header = info->arp_index |
1118                  LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
1119                  LS_64((info->permanent ? 1 : 0), I40IW_CQPSQ_MAT_PERMANENT) |
1120                  LS_64(1, I40IW_CQPSQ_MAT_ENTRYVALID) |
1121                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1122
1123         i40iw_insert_wqe_hdr(wqe, header);
1124
1125         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_ENTRY WQE",
1126                         wqe, I40IW_CQP_WQE_SIZE * 8);
1127
1128         if (post_sq)
1129                 i40iw_sc_cqp_post_sq(cqp);
1130         return 0;
1131 }
1132
1133 /**
1134  * i40iw_sc_del_arp_cache_entry - dele arp cache entry
1135  * @cqp: struct for cqp hw
1136  * @scratch: u64 saved to be used during cqp completion
1137  * @arp_index: arp index to delete arp entry
1138  * @post_sq: flag for cqp db to ring
1139  */
1140 static enum i40iw_status_code i40iw_sc_del_arp_cache_entry(
1141                                         struct i40iw_sc_cqp *cqp,
1142                                         u64 scratch,
1143                                         u16 arp_index,
1144                                         bool post_sq)
1145 {
1146         u64 *wqe;
1147         u64 header;
1148
1149         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1150         if (!wqe)
1151                 return I40IW_ERR_RING_FULL;
1152
1153         header = arp_index |
1154                  LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
1155                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1156         i40iw_insert_wqe_hdr(wqe, header);
1157
1158         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_DEL_ENTRY WQE",
1159                         wqe, I40IW_CQP_WQE_SIZE * 8);
1160
1161         if (post_sq)
1162                 i40iw_sc_cqp_post_sq(cqp);
1163         return 0;
1164 }
1165
1166 /**
1167  * i40iw_sc_query_arp_cache_entry - cqp wqe to query arp and arp index
1168  * @cqp: struct for cqp hw
1169  * @scratch: u64 saved to be used during cqp completion
1170  * @arp_index: arp index to delete arp entry
1171  * @post_sq: flag for cqp db to ring
1172  */
1173 static enum i40iw_status_code i40iw_sc_query_arp_cache_entry(
1174                                 struct i40iw_sc_cqp *cqp,
1175                                 u64 scratch,
1176                                 u16 arp_index,
1177                                 bool post_sq)
1178 {
1179         u64 *wqe;
1180         u64 header;
1181
1182         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1183         if (!wqe)
1184                 return I40IW_ERR_RING_FULL;
1185
1186         header = arp_index |
1187                  LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
1188                  LS_64(1, I40IW_CQPSQ_MAT_QUERY) |
1189                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1190
1191         i40iw_insert_wqe_hdr(wqe, header);
1192
1193         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_ARP_CACHE_ENTRY WQE",
1194                         wqe, I40IW_CQP_WQE_SIZE * 8);
1195
1196         if (post_sq)
1197                 i40iw_sc_cqp_post_sq(cqp);
1198         return 0;
1199 }
1200
1201 /**
1202  * i40iw_sc_manage_apbvt_entry - for adding and deleting apbvt entries
1203  * @cqp: struct for cqp hw
1204  * @info: info for apbvt entry to add or delete
1205  * @scratch: u64 saved to be used during cqp completion
1206  * @post_sq: flag for cqp db to ring
1207  */
1208 static enum i40iw_status_code i40iw_sc_manage_apbvt_entry(
1209                                 struct i40iw_sc_cqp *cqp,
1210                                 struct i40iw_apbvt_info *info,
1211                                 u64 scratch,
1212                                 bool post_sq)
1213 {
1214         u64 *wqe;
1215         u64 header;
1216
1217         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1218         if (!wqe)
1219                 return I40IW_ERR_RING_FULL;
1220
1221         set_64bit_val(wqe, 16, info->port);
1222
1223         header = LS_64(I40IW_CQP_OP_MANAGE_APBVT, I40IW_CQPSQ_OPCODE) |
1224                  LS_64(info->add, I40IW_CQPSQ_MAPT_ADDPORT) |
1225                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1226
1227         i40iw_insert_wqe_hdr(wqe, header);
1228
1229         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_APBVT WQE",
1230                         wqe, I40IW_CQP_WQE_SIZE * 8);
1231
1232         if (post_sq)
1233                 i40iw_sc_cqp_post_sq(cqp);
1234         return 0;
1235 }
1236
1237 /**
1238  * i40iw_sc_manage_qhash_table_entry - manage quad hash entries
1239  * @cqp: struct for cqp hw
1240  * @info: info for quad hash to manage
1241  * @scratch: u64 saved to be used during cqp completion
1242  * @post_sq: flag for cqp db to ring
1243  *
1244  * This is called before connection establishment is started. For passive connections, when
1245  * listener is created, it will call with entry type of  I40IW_QHASH_TYPE_TCP_SYN with local
1246  * ip address and tcp port. When SYN is received (passive connections) or
1247  * sent (active connections), this routine is called with entry type of
1248  * I40IW_QHASH_TYPE_TCP_ESTABLISHED and quad is passed in info.
1249  *
1250  * When iwarp connection is done and its state moves to RTS, the quad hash entry in
1251  * the hardware will point to iwarp's qp number and requires no calls from the driver.
1252  */
1253 static enum i40iw_status_code i40iw_sc_manage_qhash_table_entry(
1254                                         struct i40iw_sc_cqp *cqp,
1255                                         struct i40iw_qhash_table_info *info,
1256                                         u64 scratch,
1257                                         bool post_sq)
1258 {
1259         u64 *wqe;
1260         u64 qw1 = 0;
1261         u64 qw2 = 0;
1262         u64 temp;
1263         struct i40iw_sc_vsi *vsi = info->vsi;
1264
1265         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1266         if (!wqe)
1267                 return I40IW_ERR_RING_FULL;
1268
1269         temp = info->mac_addr[5] |
1270                 LS_64_1(info->mac_addr[4], 8) |
1271                 LS_64_1(info->mac_addr[3], 16) |
1272                 LS_64_1(info->mac_addr[2], 24) |
1273                 LS_64_1(info->mac_addr[1], 32) |
1274                 LS_64_1(info->mac_addr[0], 40);
1275
1276         set_64bit_val(wqe, 0, temp);
1277
1278         qw1 = LS_64(info->qp_num, I40IW_CQPSQ_QHASH_QPN) |
1279               LS_64(info->dest_port, I40IW_CQPSQ_QHASH_DEST_PORT);
1280         if (info->ipv4_valid) {
1281                 set_64bit_val(wqe,
1282                               48,
1283                               LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR3));
1284         } else {
1285                 set_64bit_val(wqe,
1286                               56,
1287                               LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR0) |
1288                               LS_64(info->dest_ip[1], I40IW_CQPSQ_QHASH_ADDR1));
1289
1290                 set_64bit_val(wqe,
1291                               48,
1292                               LS_64(info->dest_ip[2], I40IW_CQPSQ_QHASH_ADDR2) |
1293                               LS_64(info->dest_ip[3], I40IW_CQPSQ_QHASH_ADDR3));
1294         }
1295         qw2 = LS_64(vsi->qos[info->user_pri].qs_handle, I40IW_CQPSQ_QHASH_QS_HANDLE);
1296         if (info->vlan_valid)
1297                 qw2 |= LS_64(info->vlan_id, I40IW_CQPSQ_QHASH_VLANID);
1298         set_64bit_val(wqe, 16, qw2);
1299         if (info->entry_type == I40IW_QHASH_TYPE_TCP_ESTABLISHED) {
1300                 qw1 |= LS_64(info->src_port, I40IW_CQPSQ_QHASH_SRC_PORT);
1301                 if (!info->ipv4_valid) {
1302                         set_64bit_val(wqe,
1303                                       40,
1304                                       LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR0) |
1305                                       LS_64(info->src_ip[1], I40IW_CQPSQ_QHASH_ADDR1));
1306                         set_64bit_val(wqe,
1307                                       32,
1308                                       LS_64(info->src_ip[2], I40IW_CQPSQ_QHASH_ADDR2) |
1309                                       LS_64(info->src_ip[3], I40IW_CQPSQ_QHASH_ADDR3));
1310                 } else {
1311                         set_64bit_val(wqe,
1312                                       32,
1313                                       LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR3));
1314                 }
1315         }
1316
1317         set_64bit_val(wqe, 8, qw1);
1318         temp = LS_64(cqp->polarity, I40IW_CQPSQ_QHASH_WQEVALID) |
1319                LS_64(I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY, I40IW_CQPSQ_QHASH_OPCODE) |
1320                LS_64(info->manage, I40IW_CQPSQ_QHASH_MANAGE) |
1321                LS_64(info->ipv4_valid, I40IW_CQPSQ_QHASH_IPV4VALID) |
1322                LS_64(info->vlan_valid, I40IW_CQPSQ_QHASH_VLANVALID) |
1323                LS_64(info->entry_type, I40IW_CQPSQ_QHASH_ENTRYTYPE);
1324
1325         i40iw_insert_wqe_hdr(wqe, temp);
1326
1327         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_QHASH WQE",
1328                         wqe, I40IW_CQP_WQE_SIZE * 8);
1329
1330         if (post_sq)
1331                 i40iw_sc_cqp_post_sq(cqp);
1332         return 0;
1333 }
1334
1335 /**
1336  * i40iw_sc_alloc_local_mac_ipaddr_entry - cqp wqe for loc mac entry
1337  * @cqp: struct for cqp hw
1338  * @scratch: u64 saved to be used during cqp completion
1339  * @post_sq: flag for cqp db to ring
1340  */
1341 static enum i40iw_status_code i40iw_sc_alloc_local_mac_ipaddr_entry(
1342                                         struct i40iw_sc_cqp *cqp,
1343                                         u64 scratch,
1344                                         bool post_sq)
1345 {
1346         u64 *wqe;
1347         u64 header;
1348
1349         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1350         if (!wqe)
1351                 return I40IW_ERR_RING_FULL;
1352         header = LS_64(I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY, I40IW_CQPSQ_OPCODE) |
1353                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1354
1355         i40iw_insert_wqe_hdr(wqe, header);
1356         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ALLOCATE_LOCAL_MAC_IPADDR WQE",
1357                         wqe, I40IW_CQP_WQE_SIZE * 8);
1358         if (post_sq)
1359                 i40iw_sc_cqp_post_sq(cqp);
1360         return 0;
1361 }
1362
1363 /**
1364  * i40iw_sc_add_local_mac_ipaddr_entry - add mac enry
1365  * @cqp: struct for cqp hw
1366  * @info:mac addr info
1367  * @scratch: u64 saved to be used during cqp completion
1368  * @post_sq: flag for cqp db to ring
1369  */
1370 static enum i40iw_status_code i40iw_sc_add_local_mac_ipaddr_entry(
1371                                 struct i40iw_sc_cqp *cqp,
1372                                 struct i40iw_local_mac_ipaddr_entry_info *info,
1373                                 u64 scratch,
1374                                 bool post_sq)
1375 {
1376         u64 *wqe;
1377         u64 temp, header;
1378
1379         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1380         if (!wqe)
1381                 return I40IW_ERR_RING_FULL;
1382         temp = info->mac_addr[5] |
1383                 LS_64_1(info->mac_addr[4], 8) |
1384                 LS_64_1(info->mac_addr[3], 16) |
1385                 LS_64_1(info->mac_addr[2], 24) |
1386                 LS_64_1(info->mac_addr[1], 32) |
1387                 LS_64_1(info->mac_addr[0], 40);
1388
1389         set_64bit_val(wqe, 32, temp);
1390
1391         header = LS_64(info->entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) |
1392                  LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) |
1393                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1394
1395         i40iw_insert_wqe_hdr(wqe, header);
1396
1397         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ADD_LOCAL_MAC_IPADDR WQE",
1398                         wqe, I40IW_CQP_WQE_SIZE * 8);
1399
1400         if (post_sq)
1401                 i40iw_sc_cqp_post_sq(cqp);
1402         return 0;
1403 }
1404
1405 /**
1406  * i40iw_sc_del_local_mac_ipaddr_entry - cqp wqe to dele local mac
1407  * @cqp: struct for cqp hw
1408  * @scratch: u64 saved to be used during cqp completion
1409  * @entry_idx: index of mac entry
1410  * @ ignore_ref_count: to force mac adde delete
1411  * @post_sq: flag for cqp db to ring
1412  */
1413 static enum i40iw_status_code i40iw_sc_del_local_mac_ipaddr_entry(
1414                                 struct i40iw_sc_cqp *cqp,
1415                                 u64 scratch,
1416                                 u8 entry_idx,
1417                                 u8 ignore_ref_count,
1418                                 bool post_sq)
1419 {
1420         u64 *wqe;
1421         u64 header;
1422
1423         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1424         if (!wqe)
1425                 return I40IW_ERR_RING_FULL;
1426         header = LS_64(entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) |
1427                  LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) |
1428                  LS_64(1, I40IW_CQPSQ_MLIPA_FREEENTRY) |
1429                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
1430                  LS_64(ignore_ref_count, I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT);
1431
1432         i40iw_insert_wqe_hdr(wqe, header);
1433
1434         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "DEL_LOCAL_MAC_IPADDR WQE",
1435                         wqe, I40IW_CQP_WQE_SIZE * 8);
1436
1437         if (post_sq)
1438                 i40iw_sc_cqp_post_sq(cqp);
1439         return 0;
1440 }
1441
1442 /**
1443  * i40iw_sc_cqp_nop - send a nop wqe
1444  * @cqp: struct for cqp hw
1445  * @scratch: u64 saved to be used during cqp completion
1446  * @post_sq: flag for cqp db to ring
1447  */
1448 static enum i40iw_status_code i40iw_sc_cqp_nop(struct i40iw_sc_cqp *cqp,
1449                                                u64 scratch,
1450                                                bool post_sq)
1451 {
1452         u64 *wqe;
1453         u64 header;
1454
1455         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1456         if (!wqe)
1457                 return I40IW_ERR_RING_FULL;
1458         header = LS_64(I40IW_CQP_OP_NOP, I40IW_CQPSQ_OPCODE) |
1459                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1460         i40iw_insert_wqe_hdr(wqe, header);
1461         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "NOP WQE",
1462                         wqe, I40IW_CQP_WQE_SIZE * 8);
1463
1464         if (post_sq)
1465                 i40iw_sc_cqp_post_sq(cqp);
1466         return 0;
1467 }
1468
1469 /**
1470  * i40iw_sc_ceq_init - initialize ceq
1471  * @ceq: ceq sc structure
1472  * @info: ceq initialization info
1473  */
1474 static enum i40iw_status_code i40iw_sc_ceq_init(struct i40iw_sc_ceq *ceq,
1475                                                 struct i40iw_ceq_init_info *info)
1476 {
1477         u32 pble_obj_cnt;
1478
1479         if ((info->elem_cnt < I40IW_MIN_CEQ_ENTRIES) ||
1480             (info->elem_cnt > I40IW_MAX_CEQ_ENTRIES))
1481                 return I40IW_ERR_INVALID_SIZE;
1482
1483         if (info->ceq_id >= I40IW_MAX_CEQID)
1484                 return I40IW_ERR_INVALID_CEQ_ID;
1485
1486         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
1487
1488         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
1489                 return I40IW_ERR_INVALID_PBLE_INDEX;
1490
1491         ceq->size = sizeof(*ceq);
1492         ceq->ceqe_base = (struct i40iw_ceqe *)info->ceqe_base;
1493         ceq->ceq_id = info->ceq_id;
1494         ceq->dev = info->dev;
1495         ceq->elem_cnt = info->elem_cnt;
1496         ceq->ceq_elem_pa = info->ceqe_pa;
1497         ceq->virtual_map = info->virtual_map;
1498
1499         ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);
1500         ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);
1501         ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);
1502
1503         ceq->tph_en = info->tph_en;
1504         ceq->tph_val = info->tph_val;
1505         ceq->polarity = 1;
1506         I40IW_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);
1507         ceq->dev->ceq[info->ceq_id] = ceq;
1508
1509         return 0;
1510 }
1511
1512 /**
1513  * i40iw_sc_ceq_create - create ceq wqe
1514  * @ceq: ceq sc structure
1515  * @scratch: u64 saved to be used during cqp completion
1516  * @post_sq: flag for cqp db to ring
1517  */
1518 static enum i40iw_status_code i40iw_sc_ceq_create(struct i40iw_sc_ceq *ceq,
1519                                                   u64 scratch,
1520                                                   bool post_sq)
1521 {
1522         struct i40iw_sc_cqp *cqp;
1523         u64 *wqe;
1524         u64 header;
1525
1526         cqp = ceq->dev->cqp;
1527         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1528         if (!wqe)
1529                 return I40IW_ERR_RING_FULL;
1530         set_64bit_val(wqe, 16, ceq->elem_cnt);
1531         set_64bit_val(wqe, 32, (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));
1532         set_64bit_val(wqe, 48, (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));
1533         set_64bit_val(wqe, 56, LS_64(ceq->tph_val, I40IW_CQPSQ_TPHVAL));
1534
1535         header = ceq->ceq_id |
1536                  LS_64(I40IW_CQP_OP_CREATE_CEQ, I40IW_CQPSQ_OPCODE) |
1537                  LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) |
1538                  LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) |
1539                  LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) |
1540                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1541
1542         i40iw_insert_wqe_hdr(wqe, header);
1543
1544         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_CREATE WQE",
1545                         wqe, I40IW_CQP_WQE_SIZE * 8);
1546
1547         if (post_sq)
1548                 i40iw_sc_cqp_post_sq(cqp);
1549         return 0;
1550 }
1551
1552 /**
1553  * i40iw_sc_cceq_create_done - poll for control ceq wqe to complete
1554  * @ceq: ceq sc structure
1555  */
1556 static enum i40iw_status_code i40iw_sc_cceq_create_done(struct i40iw_sc_ceq *ceq)
1557 {
1558         struct i40iw_sc_cqp *cqp;
1559
1560         cqp = ceq->dev->cqp;
1561         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CEQ, NULL);
1562 }
1563
1564 /**
1565  * i40iw_sc_cceq_destroy_done - poll for destroy cceq to complete
1566  * @ceq: ceq sc structure
1567  */
1568 static enum i40iw_status_code i40iw_sc_cceq_destroy_done(struct i40iw_sc_ceq *ceq)
1569 {
1570         struct i40iw_sc_cqp *cqp;
1571
1572         cqp = ceq->dev->cqp;
1573         cqp->process_cqp_sds = i40iw_update_sds_noccq;
1574         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_CEQ, NULL);
1575 }
1576
1577 /**
1578  * i40iw_sc_cceq_create - create cceq
1579  * @ceq: ceq sc structure
1580  * @scratch: u64 saved to be used during cqp completion
1581  */
1582 static enum i40iw_status_code i40iw_sc_cceq_create(struct i40iw_sc_ceq *ceq, u64 scratch)
1583 {
1584         enum i40iw_status_code ret_code;
1585
1586         ret_code = i40iw_sc_ceq_create(ceq, scratch, true);
1587         if (!ret_code)
1588                 ret_code = i40iw_sc_cceq_create_done(ceq);
1589         return ret_code;
1590 }
1591
1592 /**
1593  * i40iw_sc_ceq_destroy - destroy ceq
1594  * @ceq: ceq sc structure
1595  * @scratch: u64 saved to be used during cqp completion
1596  * @post_sq: flag for cqp db to ring
1597  */
1598 static enum i40iw_status_code i40iw_sc_ceq_destroy(struct i40iw_sc_ceq *ceq,
1599                                                    u64 scratch,
1600                                                    bool post_sq)
1601 {
1602         struct i40iw_sc_cqp *cqp;
1603         u64 *wqe;
1604         u64 header;
1605
1606         cqp = ceq->dev->cqp;
1607         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1608         if (!wqe)
1609                 return I40IW_ERR_RING_FULL;
1610         set_64bit_val(wqe, 16, ceq->elem_cnt);
1611         set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx);
1612         header = ceq->ceq_id |
1613                  LS_64(I40IW_CQP_OP_DESTROY_CEQ, I40IW_CQPSQ_OPCODE) |
1614                  LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) |
1615                  LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) |
1616                  LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) |
1617                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1618         i40iw_insert_wqe_hdr(wqe, header);
1619         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_DESTROY WQE",
1620                         wqe, I40IW_CQP_WQE_SIZE * 8);
1621
1622         if (post_sq)
1623                 i40iw_sc_cqp_post_sq(cqp);
1624         return 0;
1625 }
1626
1627 /**
1628  * i40iw_sc_process_ceq - process ceq
1629  * @dev: sc device struct
1630  * @ceq: ceq sc structure
1631  */
1632 static void *i40iw_sc_process_ceq(struct i40iw_sc_dev *dev, struct i40iw_sc_ceq *ceq)
1633 {
1634         u64 temp;
1635         u64 *ceqe;
1636         struct i40iw_sc_cq *cq = NULL;
1637         u8 polarity;
1638
1639         ceqe = (u64 *)I40IW_GET_CURRENT_CEQ_ELEMENT(ceq);
1640         get_64bit_val(ceqe, 0, &temp);
1641         polarity = (u8)RS_64(temp, I40IW_CEQE_VALID);
1642         if (polarity != ceq->polarity)
1643                 return cq;
1644
1645         cq = (struct i40iw_sc_cq *)(unsigned long)LS_64_1(temp, 1);
1646
1647         I40IW_RING_MOVE_TAIL(ceq->ceq_ring);
1648         if (I40IW_RING_GETCURRENT_TAIL(ceq->ceq_ring) == 0)
1649                 ceq->polarity ^= 1;
1650
1651         if (dev->is_pf)
1652                 i40iw_wr32(dev->hw, I40E_PFPE_CQACK, cq->cq_uk.cq_id);
1653         else
1654                 i40iw_wr32(dev->hw, I40E_VFPE_CQACK1, cq->cq_uk.cq_id);
1655
1656         return cq;
1657 }
1658
1659 /**
1660  * i40iw_sc_aeq_init - initialize aeq
1661  * @aeq: aeq structure ptr
1662  * @info: aeq initialization info
1663  */
1664 static enum i40iw_status_code i40iw_sc_aeq_init(struct i40iw_sc_aeq *aeq,
1665                                                 struct i40iw_aeq_init_info *info)
1666 {
1667         u32 pble_obj_cnt;
1668
1669         if ((info->elem_cnt < I40IW_MIN_AEQ_ENTRIES) ||
1670             (info->elem_cnt > I40IW_MAX_AEQ_ENTRIES))
1671                 return I40IW_ERR_INVALID_SIZE;
1672         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
1673
1674         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
1675                 return I40IW_ERR_INVALID_PBLE_INDEX;
1676
1677         aeq->size = sizeof(*aeq);
1678         aeq->polarity = 1;
1679         aeq->aeqe_base = (struct i40iw_sc_aeqe *)info->aeqe_base;
1680         aeq->dev = info->dev;
1681         aeq->elem_cnt = info->elem_cnt;
1682
1683         aeq->aeq_elem_pa = info->aeq_elem_pa;
1684         I40IW_RING_INIT(aeq->aeq_ring, aeq->elem_cnt);
1685         info->dev->aeq = aeq;
1686
1687         aeq->virtual_map = info->virtual_map;
1688         aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL);
1689         aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0);
1690         aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0);
1691         info->dev->aeq = aeq;
1692         return 0;
1693 }
1694
1695 /**
1696  * i40iw_sc_aeq_create - create aeq
1697  * @aeq: aeq structure ptr
1698  * @scratch: u64 saved to be used during cqp completion
1699  * @post_sq: flag for cqp db to ring
1700  */
1701 static enum i40iw_status_code i40iw_sc_aeq_create(struct i40iw_sc_aeq *aeq,
1702                                                   u64 scratch,
1703                                                   bool post_sq)
1704 {
1705         u64 *wqe;
1706         struct i40iw_sc_cqp *cqp;
1707         u64 header;
1708
1709         cqp = aeq->dev->cqp;
1710         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1711         if (!wqe)
1712                 return I40IW_ERR_RING_FULL;
1713         set_64bit_val(wqe, 16, aeq->elem_cnt);
1714         set_64bit_val(wqe, 32,
1715                       (aeq->virtual_map ? 0 : aeq->aeq_elem_pa));
1716         set_64bit_val(wqe, 48,
1717                       (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0));
1718
1719         header = LS_64(I40IW_CQP_OP_CREATE_AEQ, I40IW_CQPSQ_OPCODE) |
1720                  LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) |
1721                  LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) |
1722                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1723
1724         i40iw_insert_wqe_hdr(wqe, header);
1725         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "AEQ_CREATE WQE",
1726                         wqe, I40IW_CQP_WQE_SIZE * 8);
1727         if (post_sq)
1728                 i40iw_sc_cqp_post_sq(cqp);
1729         return 0;
1730 }
1731
1732 /**
1733  * i40iw_sc_aeq_destroy - destroy aeq during close
1734  * @aeq: aeq structure ptr
1735  * @scratch: u64 saved to be used during cqp completion
1736  * @post_sq: flag for cqp db to ring
1737  */
1738 static enum i40iw_status_code i40iw_sc_aeq_destroy(struct i40iw_sc_aeq *aeq,
1739                                                    u64 scratch,
1740                                                    bool post_sq)
1741 {
1742         u64 *wqe;
1743         struct i40iw_sc_cqp *cqp;
1744         u64 header;
1745
1746         cqp = aeq->dev->cqp;
1747         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1748         if (!wqe)
1749                 return I40IW_ERR_RING_FULL;
1750         set_64bit_val(wqe, 16, aeq->elem_cnt);
1751         set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx);
1752         header = LS_64(I40IW_CQP_OP_DESTROY_AEQ, I40IW_CQPSQ_OPCODE) |
1753                  LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) |
1754                  LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) |
1755                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1756         i40iw_insert_wqe_hdr(wqe, header);
1757
1758         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "AEQ_DESTROY WQE",
1759                         wqe, I40IW_CQP_WQE_SIZE * 8);
1760         if (post_sq)
1761                 i40iw_sc_cqp_post_sq(cqp);
1762         return 0;
1763 }
1764
1765 /**
1766  * i40iw_sc_get_next_aeqe - get next aeq entry
1767  * @aeq: aeq structure ptr
1768  * @info: aeqe info to be returned
1769  */
1770 static enum i40iw_status_code i40iw_sc_get_next_aeqe(struct i40iw_sc_aeq *aeq,
1771                                                      struct i40iw_aeqe_info *info)
1772 {
1773         u64 temp, compl_ctx;
1774         u64 *aeqe;
1775         u16 wqe_idx;
1776         u8 ae_src;
1777         u8 polarity;
1778
1779         aeqe = (u64 *)I40IW_GET_CURRENT_AEQ_ELEMENT(aeq);
1780         get_64bit_val(aeqe, 0, &compl_ctx);
1781         get_64bit_val(aeqe, 8, &temp);
1782         polarity = (u8)RS_64(temp, I40IW_AEQE_VALID);
1783
1784         if (aeq->polarity != polarity)
1785                 return I40IW_ERR_QUEUE_EMPTY;
1786
1787         i40iw_debug_buf(aeq->dev, I40IW_DEBUG_WQE, "AEQ_ENTRY", aeqe, 16);
1788
1789         ae_src = (u8)RS_64(temp, I40IW_AEQE_AESRC);
1790         wqe_idx = (u16)RS_64(temp, I40IW_AEQE_WQDESCIDX);
1791         info->qp_cq_id = (u32)RS_64(temp, I40IW_AEQE_QPCQID);
1792         info->ae_id = (u16)RS_64(temp, I40IW_AEQE_AECODE);
1793         info->tcp_state = (u8)RS_64(temp, I40IW_AEQE_TCPSTATE);
1794         info->iwarp_state = (u8)RS_64(temp, I40IW_AEQE_IWSTATE);
1795         info->q2_data_written = (u8)RS_64(temp, I40IW_AEQE_Q2DATA);
1796         info->aeqe_overflow = (bool)RS_64(temp, I40IW_AEQE_OVERFLOW);
1797
1798         switch (info->ae_id) {
1799         case I40IW_AE_PRIV_OPERATION_DENIED:
1800         case I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG:
1801         case I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT:
1802         case I40IW_AE_BAD_CLOSE:
1803         case I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE:
1804         case I40IW_AE_RDMA_READ_WHILE_ORD_ZERO:
1805         case I40IW_AE_STAG_ZERO_INVALID:
1806         case I40IW_AE_IB_RREQ_AND_Q1_FULL:
1807         case I40IW_AE_WQE_UNEXPECTED_OPCODE:
1808         case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:
1809         case I40IW_AE_DDP_UBE_INVALID_MO:
1810         case I40IW_AE_DDP_UBE_INVALID_QN:
1811         case I40IW_AE_DDP_NO_L_BIT:
1812         case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
1813         case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
1814         case I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST:
1815         case I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
1816         case I40IW_AE_INVALID_ARP_ENTRY:
1817         case I40IW_AE_INVALID_TCP_OPTION_RCVD:
1818         case I40IW_AE_STALE_ARP_ENTRY:
1819         case I40IW_AE_LLP_CLOSE_COMPLETE:
1820         case I40IW_AE_LLP_CONNECTION_RESET:
1821         case I40IW_AE_LLP_FIN_RECEIVED:
1822         case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:
1823         case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
1824         case I40IW_AE_LLP_SYN_RECEIVED:
1825         case I40IW_AE_LLP_TERMINATE_RECEIVED:
1826         case I40IW_AE_LLP_TOO_MANY_RETRIES:
1827         case I40IW_AE_LLP_DOUBT_REACHABILITY:
1828         case I40IW_AE_RESET_SENT:
1829         case I40IW_AE_TERMINATE_SENT:
1830         case I40IW_AE_RESET_NOT_SENT:
1831         case I40IW_AE_LCE_QP_CATASTROPHIC:
1832         case I40IW_AE_QP_SUSPEND_COMPLETE:
1833                 info->qp = true;
1834                 info->compl_ctx = compl_ctx;
1835                 ae_src = I40IW_AE_SOURCE_RSVD;
1836                 break;
1837         case I40IW_AE_LCE_CQ_CATASTROPHIC:
1838                 info->cq = true;
1839                 info->compl_ctx = LS_64_1(compl_ctx, 1);
1840                 ae_src = I40IW_AE_SOURCE_RSVD;
1841                 break;
1842         }
1843
1844         switch (ae_src) {
1845         case I40IW_AE_SOURCE_RQ:
1846         case I40IW_AE_SOURCE_RQ_0011:
1847                 info->qp = true;
1848                 info->wqe_idx = wqe_idx;
1849                 info->compl_ctx = compl_ctx;
1850                 break;
1851         case I40IW_AE_SOURCE_CQ:
1852         case I40IW_AE_SOURCE_CQ_0110:
1853         case I40IW_AE_SOURCE_CQ_1010:
1854         case I40IW_AE_SOURCE_CQ_1110:
1855                 info->cq = true;
1856                 info->compl_ctx = LS_64_1(compl_ctx, 1);
1857                 break;
1858         case I40IW_AE_SOURCE_SQ:
1859         case I40IW_AE_SOURCE_SQ_0111:
1860                 info->qp = true;
1861                 info->sq = true;
1862                 info->wqe_idx = wqe_idx;
1863                 info->compl_ctx = compl_ctx;
1864                 break;
1865         case I40IW_AE_SOURCE_IN_RR_WR:
1866         case I40IW_AE_SOURCE_IN_RR_WR_1011:
1867                 info->qp = true;
1868                 info->compl_ctx = compl_ctx;
1869                 info->in_rdrsp_wr = true;
1870                 break;
1871         case I40IW_AE_SOURCE_OUT_RR:
1872         case I40IW_AE_SOURCE_OUT_RR_1111:
1873                 info->qp = true;
1874                 info->compl_ctx = compl_ctx;
1875                 info->out_rdrsp = true;
1876                 break;
1877         case I40IW_AE_SOURCE_RSVD:
1878                 /* fallthrough */
1879         default:
1880                 break;
1881         }
1882         I40IW_RING_MOVE_TAIL(aeq->aeq_ring);
1883         if (I40IW_RING_GETCURRENT_TAIL(aeq->aeq_ring) == 0)
1884                 aeq->polarity ^= 1;
1885         return 0;
1886 }
1887
1888 /**
1889  * i40iw_sc_repost_aeq_entries - repost completed aeq entries
1890  * @dev: sc device struct
1891  * @count: allocate count
1892  */
1893 static enum i40iw_status_code i40iw_sc_repost_aeq_entries(struct i40iw_sc_dev *dev,
1894                                                           u32 count)
1895 {
1896
1897         if (dev->is_pf)
1898                 i40iw_wr32(dev->hw, I40E_PFPE_AEQALLOC, count);
1899         else
1900                 i40iw_wr32(dev->hw, I40E_VFPE_AEQALLOC1, count);
1901
1902         return 0;
1903 }
1904
1905 /**
1906  * i40iw_sc_aeq_create_done - create aeq
1907  * @aeq: aeq structure ptr
1908  */
1909 static enum i40iw_status_code i40iw_sc_aeq_create_done(struct i40iw_sc_aeq *aeq)
1910 {
1911         struct i40iw_sc_cqp *cqp;
1912
1913         cqp = aeq->dev->cqp;
1914         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_AEQ, NULL);
1915 }
1916
1917 /**
1918  * i40iw_sc_aeq_destroy_done - destroy of aeq during close
1919  * @aeq: aeq structure ptr
1920  */
1921 static enum i40iw_status_code i40iw_sc_aeq_destroy_done(struct i40iw_sc_aeq *aeq)
1922 {
1923         struct i40iw_sc_cqp *cqp;
1924
1925         cqp = aeq->dev->cqp;
1926         return  i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_AEQ, NULL);
1927 }
1928
1929 /**
1930  * i40iw_sc_ccq_init - initialize control cq
1931  * @cq: sc's cq ctruct
1932  * @info: info for control cq initialization
1933  */
1934 static enum i40iw_status_code i40iw_sc_ccq_init(struct i40iw_sc_cq *cq,
1935                                                 struct i40iw_ccq_init_info *info)
1936 {
1937         u32 pble_obj_cnt;
1938
1939         if (info->num_elem < I40IW_MIN_CQ_SIZE || info->num_elem > I40IW_MAX_CQ_SIZE)
1940                 return I40IW_ERR_INVALID_SIZE;
1941
1942         if (info->ceq_id > I40IW_MAX_CEQID)
1943                 return I40IW_ERR_INVALID_CEQ_ID;
1944
1945         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
1946
1947         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
1948                 return I40IW_ERR_INVALID_PBLE_INDEX;
1949
1950         cq->cq_pa = info->cq_pa;
1951         cq->cq_uk.cq_base = info->cq_base;
1952         cq->shadow_area_pa = info->shadow_area_pa;
1953         cq->cq_uk.shadow_area = info->shadow_area;
1954         cq->shadow_read_threshold = info->shadow_read_threshold;
1955         cq->dev = info->dev;
1956         cq->ceq_id = info->ceq_id;
1957         cq->cq_uk.cq_size = info->num_elem;
1958         cq->cq_type = I40IW_CQ_TYPE_CQP;
1959         cq->ceqe_mask = info->ceqe_mask;
1960         I40IW_RING_INIT(cq->cq_uk.cq_ring, info->num_elem);
1961
1962         cq->cq_uk.cq_id = 0;    /* control cq is id 0 always */
1963         cq->ceq_id_valid = info->ceq_id_valid;
1964         cq->tph_en = info->tph_en;
1965         cq->tph_val = info->tph_val;
1966         cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct;
1967
1968         cq->pbl_list = info->pbl_list;
1969         cq->virtual_map = info->virtual_map;
1970         cq->pbl_chunk_size = info->pbl_chunk_size;
1971         cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
1972         cq->cq_uk.polarity = true;
1973
1974         /* following are only for iw cqs so initialize them to zero */
1975         cq->cq_uk.cqe_alloc_reg = NULL;
1976         info->dev->ccq = cq;
1977         return 0;
1978 }
1979
1980 /**
1981  * i40iw_sc_ccq_create_done - poll cqp for ccq create
1982  * @ccq: ccq sc struct
1983  */
1984 static enum i40iw_status_code i40iw_sc_ccq_create_done(struct i40iw_sc_cq *ccq)
1985 {
1986         struct i40iw_sc_cqp *cqp;
1987
1988         cqp = ccq->dev->cqp;
1989         return  i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CQ, NULL);
1990 }
1991
1992 /**
1993  * i40iw_sc_ccq_create - create control cq
1994  * @ccq: ccq sc struct
1995  * @scratch: u64 saved to be used during cqp completion
1996  * @check_overflow: overlow flag for ccq
1997  * @post_sq: flag for cqp db to ring
1998  */
1999 static enum i40iw_status_code i40iw_sc_ccq_create(struct i40iw_sc_cq *ccq,
2000                                                   u64 scratch,
2001                                                   bool check_overflow,
2002                                                   bool post_sq)
2003 {
2004         u64 *wqe;
2005         struct i40iw_sc_cqp *cqp;
2006         u64 header;
2007         enum i40iw_status_code ret_code;
2008
2009         cqp = ccq->dev->cqp;
2010         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2011         if (!wqe)
2012                 return I40IW_ERR_RING_FULL;
2013         set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
2014         set_64bit_val(wqe, 8, RS_64_1(ccq, 1));
2015         set_64bit_val(wqe, 16,
2016                       LS_64(ccq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
2017         set_64bit_val(wqe, 32, (ccq->virtual_map ? 0 : ccq->cq_pa));
2018         set_64bit_val(wqe, 40, ccq->shadow_area_pa);
2019         set_64bit_val(wqe, 48,
2020                       (ccq->virtual_map ? ccq->first_pm_pbl_idx : 0));
2021         set_64bit_val(wqe, 56,
2022                       LS_64(ccq->tph_val, I40IW_CQPSQ_TPHVAL));
2023
2024         header = ccq->cq_uk.cq_id |
2025                  LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2026                  LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |
2027                  LS_64(ccq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2028                  LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
2029                  LS_64(ccq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2030                  LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2031                  LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2032                  LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) |
2033                  LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2034                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2035
2036         i40iw_insert_wqe_hdr(wqe, header);
2037
2038         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_CREATE WQE",
2039                         wqe, I40IW_CQP_WQE_SIZE * 8);
2040
2041         if (post_sq) {
2042                 i40iw_sc_cqp_post_sq(cqp);
2043                 ret_code = i40iw_sc_ccq_create_done(ccq);
2044                 if (ret_code)
2045                         return ret_code;
2046         }
2047         cqp->process_cqp_sds = i40iw_cqp_sds_cmd;
2048
2049         return 0;
2050 }
2051
2052 /**
2053  * i40iw_sc_ccq_destroy - destroy ccq during close
2054  * @ccq: ccq sc struct
2055  * @scratch: u64 saved to be used during cqp completion
2056  * @post_sq: flag for cqp db to ring
2057  */
2058 static enum i40iw_status_code i40iw_sc_ccq_destroy(struct i40iw_sc_cq *ccq,
2059                                                    u64 scratch,
2060                                                    bool post_sq)
2061 {
2062         struct i40iw_sc_cqp *cqp;
2063         u64 *wqe;
2064         u64 header;
2065         enum i40iw_status_code ret_code = 0;
2066         u32 tail, val, error;
2067
2068         cqp = ccq->dev->cqp;
2069         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2070         if (!wqe)
2071                 return I40IW_ERR_RING_FULL;
2072         set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
2073         set_64bit_val(wqe, 8, RS_64_1(ccq, 1));
2074         set_64bit_val(wqe, 40, ccq->shadow_area_pa);
2075
2076         header = ccq->cq_uk.cq_id |
2077                  LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2078                  LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) |
2079                  LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2080                  LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2081                  LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) |
2082                  LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2083                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2084
2085         i40iw_insert_wqe_hdr(wqe, header);
2086
2087         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_DESTROY WQE",
2088                         wqe, I40IW_CQP_WQE_SIZE * 8);
2089
2090         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
2091         if (error)
2092                 return I40IW_ERR_CQP_COMPL_ERROR;
2093
2094         if (post_sq) {
2095                 i40iw_sc_cqp_post_sq(cqp);
2096                 ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);
2097         }
2098
2099         cqp->process_cqp_sds = i40iw_update_sds_noccq;
2100
2101         return ret_code;
2102 }
2103
2104 /**
2105  * i40iw_sc_cq_init - initialize completion q
2106  * @cq: cq struct
2107  * @info: cq initialization info
2108  */
2109 static enum i40iw_status_code i40iw_sc_cq_init(struct i40iw_sc_cq *cq,
2110                                                struct i40iw_cq_init_info *info)
2111 {
2112         u32 __iomem *cqe_alloc_reg = NULL;
2113         enum i40iw_status_code ret_code;
2114         u32 pble_obj_cnt;
2115         u32 arm_offset;
2116
2117         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2118
2119         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
2120                 return I40IW_ERR_INVALID_PBLE_INDEX;
2121
2122         cq->cq_pa = info->cq_base_pa;
2123         cq->dev = info->dev;
2124         cq->ceq_id = info->ceq_id;
2125         arm_offset = (info->dev->is_pf) ? I40E_PFPE_CQARM : I40E_VFPE_CQARM1;
2126         if (i40iw_get_hw_addr(cq->dev))
2127                 cqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(cq->dev) +
2128                                               arm_offset);
2129         info->cq_uk_init_info.cqe_alloc_reg = cqe_alloc_reg;
2130         ret_code = i40iw_cq_uk_init(&cq->cq_uk, &info->cq_uk_init_info);
2131         if (ret_code)
2132                 return ret_code;
2133         cq->virtual_map = info->virtual_map;
2134         cq->pbl_chunk_size = info->pbl_chunk_size;
2135         cq->ceqe_mask = info->ceqe_mask;
2136         cq->cq_type = (info->type) ? info->type : I40IW_CQ_TYPE_IWARP;
2137
2138         cq->shadow_area_pa = info->shadow_area_pa;
2139         cq->shadow_read_threshold = info->shadow_read_threshold;
2140
2141         cq->ceq_id_valid = info->ceq_id_valid;
2142         cq->tph_en = info->tph_en;
2143         cq->tph_val = info->tph_val;
2144
2145         cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2146
2147         return 0;
2148 }
2149
2150 /**
2151  * i40iw_sc_cq_create - create completion q
2152  * @cq: cq struct
2153  * @scratch: u64 saved to be used during cqp completion
2154  * @check_overflow: flag for overflow check
2155  * @post_sq: flag for cqp db to ring
2156  */
2157 static enum i40iw_status_code i40iw_sc_cq_create(struct i40iw_sc_cq *cq,
2158                                                  u64 scratch,
2159                                                  bool check_overflow,
2160                                                  bool post_sq)
2161 {
2162         u64 *wqe;
2163         struct i40iw_sc_cqp *cqp;
2164         u64 header;
2165
2166         if (cq->cq_uk.cq_id > I40IW_MAX_CQID)
2167                 return I40IW_ERR_INVALID_CQ_ID;
2168
2169         if (cq->ceq_id > I40IW_MAX_CEQID)
2170                 return I40IW_ERR_INVALID_CEQ_ID;
2171
2172         cqp = cq->dev->cqp;
2173         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2174         if (!wqe)
2175                 return I40IW_ERR_RING_FULL;
2176
2177         set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2178         set_64bit_val(wqe, 8, RS_64_1(cq, 1));
2179         set_64bit_val(wqe,
2180                       16,
2181                       LS_64(cq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
2182
2183         set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
2184
2185         set_64bit_val(wqe, 40, cq->shadow_area_pa);
2186         set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2187         set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL));
2188
2189         header = cq->cq_uk.cq_id |
2190                  LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2191                  LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |
2192                  LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2193                  LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
2194                  LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2195                  LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2196                  LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2197                  LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
2198                  LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2199                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2200
2201         i40iw_insert_wqe_hdr(wqe, header);
2202
2203         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_CREATE WQE",
2204                         wqe, I40IW_CQP_WQE_SIZE * 8);
2205
2206         if (post_sq)
2207                 i40iw_sc_cqp_post_sq(cqp);
2208         return 0;
2209 }
2210
2211 /**
2212  * i40iw_sc_cq_destroy - destroy completion q
2213  * @cq: cq struct
2214  * @scratch: u64 saved to be used during cqp completion
2215  * @post_sq: flag for cqp db to ring
2216  */
2217 static enum i40iw_status_code i40iw_sc_cq_destroy(struct i40iw_sc_cq *cq,
2218                                                   u64 scratch,
2219                                                   bool post_sq)
2220 {
2221         struct i40iw_sc_cqp *cqp;
2222         u64 *wqe;
2223         u64 header;
2224
2225         cqp = cq->dev->cqp;
2226         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2227         if (!wqe)
2228                 return I40IW_ERR_RING_FULL;
2229         set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2230         set_64bit_val(wqe, 8, RS_64_1(cq, 1));
2231         set_64bit_val(wqe, 40, cq->shadow_area_pa);
2232         set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2233
2234         header = cq->cq_uk.cq_id |
2235                  LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2236                  LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) |
2237                  LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2238                  LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2239                  LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2240                  LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2241                  LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
2242                  LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2243                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2244
2245         i40iw_insert_wqe_hdr(wqe, header);
2246
2247         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_DESTROY WQE",
2248                         wqe, I40IW_CQP_WQE_SIZE * 8);
2249
2250         if (post_sq)
2251                 i40iw_sc_cqp_post_sq(cqp);
2252         return 0;
2253 }
2254
2255 /**
2256  * i40iw_sc_cq_modify - modify a Completion Queue
2257  * @cq: cq struct
2258  * @info: modification info struct
2259  * @scratch:
2260  * @post_sq: flag to post to sq
2261  */
2262 static enum i40iw_status_code i40iw_sc_cq_modify(struct i40iw_sc_cq *cq,
2263                                                  struct i40iw_modify_cq_info *info,
2264                                                  u64 scratch,
2265                                                  bool post_sq)
2266 {
2267         struct i40iw_sc_cqp *cqp;
2268         u64 *wqe;
2269         u64 header;
2270         u32 cq_size, ceq_id, first_pm_pbl_idx;
2271         u8 pbl_chunk_size;
2272         bool virtual_map, ceq_id_valid, check_overflow;
2273         u32 pble_obj_cnt;
2274
2275         if (info->ceq_valid && (info->ceq_id > I40IW_MAX_CEQID))
2276                 return I40IW_ERR_INVALID_CEQ_ID;
2277
2278         pble_obj_cnt = cq->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2279
2280         if (info->cq_resize && info->virtual_map &&
2281             (info->first_pm_pbl_idx >= pble_obj_cnt))
2282                 return I40IW_ERR_INVALID_PBLE_INDEX;
2283
2284         cqp = cq->dev->cqp;
2285         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2286         if (!wqe)
2287                 return I40IW_ERR_RING_FULL;
2288
2289         cq->pbl_list = info->pbl_list;
2290         cq->cq_pa = info->cq_pa;
2291         cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2292
2293         cq_size = info->cq_resize ? info->cq_size : cq->cq_uk.cq_size;
2294         if (info->ceq_change) {
2295                 ceq_id_valid = true;
2296                 ceq_id = info->ceq_id;
2297         } else {
2298                 ceq_id_valid = cq->ceq_id_valid;
2299                 ceq_id = ceq_id_valid ? cq->ceq_id : 0;
2300         }
2301         virtual_map = info->cq_resize ? info->virtual_map : cq->virtual_map;
2302         first_pm_pbl_idx = (info->cq_resize ?
2303                             (info->virtual_map ? info->first_pm_pbl_idx : 0) :
2304                             (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2305         pbl_chunk_size = (info->cq_resize ?
2306                           (info->virtual_map ? info->pbl_chunk_size : 0) :
2307                           (cq->virtual_map ? cq->pbl_chunk_size : 0));
2308         check_overflow = info->check_overflow_change ? info->check_overflow :
2309                          cq->check_overflow;
2310         cq->cq_uk.cq_size = cq_size;
2311         cq->ceq_id_valid = ceq_id_valid;
2312         cq->ceq_id = ceq_id;
2313         cq->virtual_map = virtual_map;
2314         cq->first_pm_pbl_idx = first_pm_pbl_idx;
2315         cq->pbl_chunk_size = pbl_chunk_size;
2316         cq->check_overflow = check_overflow;
2317
2318         set_64bit_val(wqe, 0, cq_size);
2319         set_64bit_val(wqe, 8, RS_64_1(cq, 1));
2320         set_64bit_val(wqe, 16,
2321                       LS_64(info->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
2322         set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
2323         set_64bit_val(wqe, 40, cq->shadow_area_pa);
2324         set_64bit_val(wqe, 48, (cq->virtual_map ? first_pm_pbl_idx : 0));
2325         set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL));
2326
2327         header = cq->cq_uk.cq_id |
2328                  LS_64(ceq_id, I40IW_CQPSQ_CQ_CEQID) |
2329                  LS_64(I40IW_CQP_OP_MODIFY_CQ, I40IW_CQPSQ_OPCODE) |
2330                  LS_64(info->cq_resize, I40IW_CQPSQ_CQ_CQRESIZE) |
2331                  LS_64(pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2332                  LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
2333                  LS_64(virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2334                  LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2335                  LS_64(ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2336                  LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
2337                  LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2338                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2339
2340         i40iw_insert_wqe_hdr(wqe, header);
2341
2342         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_MODIFY WQE",
2343                         wqe, I40IW_CQP_WQE_SIZE * 8);
2344
2345         if (post_sq)
2346                 i40iw_sc_cqp_post_sq(cqp);
2347         return 0;
2348 }
2349
2350 /**
2351  * i40iw_sc_qp_init - initialize qp
2352  * @qp: sc qp
2353  * @info: initialization qp info
2354  */
2355 static enum i40iw_status_code i40iw_sc_qp_init(struct i40iw_sc_qp *qp,
2356                                                struct i40iw_qp_init_info *info)
2357 {
2358         u32 __iomem *wqe_alloc_reg = NULL;
2359         enum i40iw_status_code ret_code;
2360         u32 pble_obj_cnt;
2361         u8 wqe_size;
2362         u32 offset;
2363
2364         qp->dev = info->pd->dev;
2365         qp->vsi = info->vsi;
2366         qp->sq_pa = info->sq_pa;
2367         qp->rq_pa = info->rq_pa;
2368         qp->hw_host_ctx_pa = info->host_ctx_pa;
2369         qp->q2_pa = info->q2_pa;
2370         qp->shadow_area_pa = info->shadow_area_pa;
2371
2372         qp->q2_buf = info->q2;
2373         qp->pd = info->pd;
2374         qp->hw_host_ctx = info->host_ctx;
2375         offset = (qp->pd->dev->is_pf) ? I40E_PFPE_WQEALLOC : I40E_VFPE_WQEALLOC1;
2376         if (i40iw_get_hw_addr(qp->pd->dev))
2377                 wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) +
2378                                               offset);
2379
2380         info->qp_uk_init_info.wqe_alloc_reg = wqe_alloc_reg;
2381         info->qp_uk_init_info.abi_ver = qp->pd->abi_ver;
2382         ret_code = i40iw_qp_uk_init(&qp->qp_uk, &info->qp_uk_init_info);
2383         if (ret_code)
2384                 return ret_code;
2385         qp->virtual_map = info->virtual_map;
2386
2387         pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2388
2389         if ((info->virtual_map && (info->sq_pa >= pble_obj_cnt)) ||
2390             (info->virtual_map && (info->rq_pa >= pble_obj_cnt)))
2391                 return I40IW_ERR_INVALID_PBLE_INDEX;
2392
2393         qp->llp_stream_handle = (void *)(-1);
2394         qp->qp_type = (info->type) ? info->type : I40IW_QP_TYPE_IWARP;
2395
2396         qp->hw_sq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.sq_ring.size,
2397                                                     false);
2398         i40iw_debug(qp->dev, I40IW_DEBUG_WQE, "%s: hw_sq_size[%04d] sq_ring.size[%04d]\n",
2399                     __func__, qp->hw_sq_size, qp->qp_uk.sq_ring.size);
2400
2401         switch (qp->pd->abi_ver) {
2402         case 4:
2403                 ret_code = i40iw_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt,
2404                                                        &wqe_size);
2405                 if (ret_code)
2406                         return ret_code;
2407                 break;
2408         case 5: /* fallthrough until next ABI version */
2409         default:
2410                 if (qp->qp_uk.max_rq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT)
2411                         return I40IW_ERR_INVALID_FRAG_COUNT;
2412                 wqe_size = I40IW_MAX_WQE_SIZE_RQ;
2413                 break;
2414         }
2415         qp->hw_rq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.rq_size *
2416                                 (wqe_size / I40IW_QP_WQE_MIN_SIZE), false);
2417         i40iw_debug(qp->dev, I40IW_DEBUG_WQE,
2418                     "%s: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",
2419                     __func__, qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size);
2420         qp->sq_tph_val = info->sq_tph_val;
2421         qp->rq_tph_val = info->rq_tph_val;
2422         qp->sq_tph_en = info->sq_tph_en;
2423         qp->rq_tph_en = info->rq_tph_en;
2424         qp->rcv_tph_en = info->rcv_tph_en;
2425         qp->xmit_tph_en = info->xmit_tph_en;
2426         qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;
2427
2428         return 0;
2429 }
2430
2431 /**
2432  * i40iw_sc_qp_create - create qp
2433  * @qp: sc qp
2434  * @info: qp create info
2435  * @scratch: u64 saved to be used during cqp completion
2436  * @post_sq: flag for cqp db to ring
2437  */
2438 static enum i40iw_status_code i40iw_sc_qp_create(
2439                                 struct i40iw_sc_qp *qp,
2440                                 struct i40iw_create_qp_info *info,
2441                                 u64 scratch,
2442                                 bool post_sq)
2443 {
2444         struct i40iw_sc_cqp *cqp;
2445         u64 *wqe;
2446         u64 header;
2447
2448         if ((qp->qp_uk.qp_id < I40IW_MIN_IW_QP_ID) ||
2449             (qp->qp_uk.qp_id > I40IW_MAX_IW_QP_ID))
2450                 return I40IW_ERR_INVALID_QP_ID;
2451
2452         cqp = qp->pd->dev->cqp;
2453         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2454         if (!wqe)
2455                 return I40IW_ERR_RING_FULL;
2456
2457         set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
2458
2459         set_64bit_val(wqe, 40, qp->shadow_area_pa);
2460
2461         header = qp->qp_uk.qp_id |
2462                  LS_64(I40IW_CQP_OP_CREATE_QP, I40IW_CQPSQ_OPCODE) |
2463                  LS_64((info->ord_valid ? 1 : 0), I40IW_CQPSQ_QP_ORDVALID) |
2464                  LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) |
2465                  LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
2466                  LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) |
2467                  LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) |
2468                  LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) |
2469                  LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) |
2470                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2471
2472         i40iw_insert_wqe_hdr(wqe, header);
2473         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_CREATE WQE",
2474                         wqe, I40IW_CQP_WQE_SIZE * 8);
2475
2476         if (post_sq)
2477                 i40iw_sc_cqp_post_sq(cqp);
2478         return 0;
2479 }
2480
2481 /**
2482  * i40iw_sc_qp_modify - modify qp cqp wqe
2483  * @qp: sc qp
2484  * @info: modify qp info
2485  * @scratch: u64 saved to be used during cqp completion
2486  * @post_sq: flag for cqp db to ring
2487  */
2488 static enum i40iw_status_code i40iw_sc_qp_modify(
2489                                 struct i40iw_sc_qp *qp,
2490                                 struct i40iw_modify_qp_info *info,
2491                                 u64 scratch,
2492                                 bool post_sq)
2493 {
2494         u64 *wqe;
2495         struct i40iw_sc_cqp *cqp;
2496         u64 header;
2497         u8 term_actions = 0;
2498         u8 term_len = 0;
2499
2500         cqp = qp->pd->dev->cqp;
2501         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2502         if (!wqe)
2503                 return I40IW_ERR_RING_FULL;
2504         if (info->next_iwarp_state == I40IW_QP_STATE_TERMINATE) {
2505                 if (info->dont_send_fin)
2506                         term_actions += I40IWQP_TERM_SEND_TERM_ONLY;
2507                 if (info->dont_send_term)
2508                         term_actions += I40IWQP_TERM_SEND_FIN_ONLY;
2509                 if ((term_actions == I40IWQP_TERM_SEND_TERM_AND_FIN) ||
2510                     (term_actions == I40IWQP_TERM_SEND_TERM_ONLY))
2511                         term_len = info->termlen;
2512         }
2513
2514         set_64bit_val(wqe,
2515                       8,
2516                       LS_64(term_len, I40IW_CQPSQ_QP_TERMLEN));
2517
2518         set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
2519         set_64bit_val(wqe, 40, qp->shadow_area_pa);
2520
2521         header = qp->qp_uk.qp_id |
2522                  LS_64(I40IW_CQP_OP_MODIFY_QP, I40IW_CQPSQ_OPCODE) |
2523                  LS_64(info->ord_valid, I40IW_CQPSQ_QP_ORDVALID) |
2524                  LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) |
2525                  LS_64(info->cached_var_valid, I40IW_CQPSQ_QP_CACHEDVARVALID) |
2526                  LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) |
2527                  LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) |
2528                  LS_64(info->force_loopback, I40IW_CQPSQ_QP_FORCELOOPBACK) |
2529                  LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
2530                  LS_64(info->remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) |
2531                  LS_64(term_actions, I40IW_CQPSQ_QP_TERMACT) |
2532                  LS_64(info->reset_tcp_conn, I40IW_CQPSQ_QP_RESETCON) |
2533                  LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) |
2534                  LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) |
2535                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2536
2537         i40iw_insert_wqe_hdr(wqe, header);
2538
2539         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_MODIFY WQE",
2540                         wqe, I40IW_CQP_WQE_SIZE * 8);
2541
2542         if (post_sq)
2543                 i40iw_sc_cqp_post_sq(cqp);
2544         return 0;
2545 }
2546
2547 /**
2548  * i40iw_sc_qp_destroy - cqp destroy qp
2549  * @qp: sc qp
2550  * @scratch: u64 saved to be used during cqp completion
2551  * @remove_hash_idx: flag if to remove hash idx
2552  * @ignore_mw_bnd: memory window bind flag
2553  * @post_sq: flag for cqp db to ring
2554  */
2555 static enum i40iw_status_code i40iw_sc_qp_destroy(
2556                                         struct i40iw_sc_qp *qp,
2557                                         u64 scratch,
2558                                         bool remove_hash_idx,
2559                                         bool ignore_mw_bnd,
2560                                         bool post_sq)
2561 {
2562         u64 *wqe;
2563         struct i40iw_sc_cqp *cqp;
2564         u64 header;
2565
2566         i40iw_qp_rem_qos(qp);
2567         cqp = qp->pd->dev->cqp;
2568         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2569         if (!wqe)
2570                 return I40IW_ERR_RING_FULL;
2571         set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
2572         set_64bit_val(wqe, 40, qp->shadow_area_pa);
2573
2574         header = qp->qp_uk.qp_id |
2575                  LS_64(I40IW_CQP_OP_DESTROY_QP, I40IW_CQPSQ_OPCODE) |
2576                  LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
2577                  LS_64(ignore_mw_bnd, I40IW_CQPSQ_QP_IGNOREMWBOUND) |
2578                  LS_64(remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) |
2579                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2580
2581         i40iw_insert_wqe_hdr(wqe, header);
2582         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_DESTROY WQE",
2583                         wqe, I40IW_CQP_WQE_SIZE * 8);
2584
2585         if (post_sq)
2586                 i40iw_sc_cqp_post_sq(cqp);
2587         return 0;
2588 }
2589
2590 /**
2591  * i40iw_sc_qp_flush_wqes - flush qp's wqe
2592  * @qp: sc qp
2593  * @info: dlush information
2594  * @scratch: u64 saved to be used during cqp completion
2595  * @post_sq: flag for cqp db to ring
2596  */
2597 static enum i40iw_status_code i40iw_sc_qp_flush_wqes(
2598                                 struct i40iw_sc_qp *qp,
2599                                 struct i40iw_qp_flush_info *info,
2600                                 u64 scratch,
2601                                 bool post_sq)
2602 {
2603         u64 temp = 0;
2604         u64 *wqe;
2605         struct i40iw_sc_cqp *cqp;
2606         u64 header;
2607         bool flush_sq = false, flush_rq = false;
2608
2609         if (info->rq && !qp->flush_rq)
2610                 flush_rq = true;
2611
2612         if (info->sq && !qp->flush_sq)
2613                 flush_sq = true;
2614
2615         qp->flush_sq |= flush_sq;
2616         qp->flush_rq |= flush_rq;
2617         if (!flush_sq && !flush_rq)
2618                 return 0;
2619
2620         cqp = qp->pd->dev->cqp;
2621         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2622         if (!wqe)
2623                 return I40IW_ERR_RING_FULL;
2624         if (info->userflushcode) {
2625                 if (flush_rq) {
2626                         temp |= LS_64(info->rq_minor_code, I40IW_CQPSQ_FWQE_RQMNERR) |
2627                                 LS_64(info->rq_major_code, I40IW_CQPSQ_FWQE_RQMJERR);
2628                 }
2629                 if (flush_sq) {
2630                         temp |= LS_64(info->sq_minor_code, I40IW_CQPSQ_FWQE_SQMNERR) |
2631                                 LS_64(info->sq_major_code, I40IW_CQPSQ_FWQE_SQMJERR);
2632                 }
2633         }
2634         set_64bit_val(wqe, 16, temp);
2635
2636         temp = (info->generate_ae) ?
2637                 info->ae_code | LS_64(info->ae_source, I40IW_CQPSQ_FWQE_AESOURCE) : 0;
2638
2639         set_64bit_val(wqe, 8, temp);
2640
2641         header = qp->qp_uk.qp_id |
2642                  LS_64(I40IW_CQP_OP_FLUSH_WQES, I40IW_CQPSQ_OPCODE) |
2643                  LS_64(info->generate_ae, I40IW_CQPSQ_FWQE_GENERATE_AE) |
2644                  LS_64(info->userflushcode, I40IW_CQPSQ_FWQE_USERFLCODE) |
2645                  LS_64(flush_sq, I40IW_CQPSQ_FWQE_FLUSHSQ) |
2646                  LS_64(flush_rq, I40IW_CQPSQ_FWQE_FLUSHRQ) |
2647                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2648
2649         i40iw_insert_wqe_hdr(wqe, header);
2650
2651         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_FLUSH WQE",
2652                         wqe, I40IW_CQP_WQE_SIZE * 8);
2653
2654         if (post_sq)
2655                 i40iw_sc_cqp_post_sq(cqp);
2656         return 0;
2657 }
2658
2659 /**
2660  * i40iw_sc_gen_ae - generate AE, currently uses flush WQE CQP OP
2661  * @qp: sc qp
2662  * @info: gen ae information
2663  * @scratch: u64 saved to be used during cqp completion
2664  * @post_sq: flag for cqp db to ring
2665  */
2666 static enum i40iw_status_code i40iw_sc_gen_ae(
2667                                 struct i40iw_sc_qp *qp,
2668                                 struct i40iw_gen_ae_info *info,
2669                                 u64 scratch,
2670                                 bool post_sq)
2671 {
2672         u64 temp;
2673         u64 *wqe;
2674         struct i40iw_sc_cqp *cqp;
2675         u64 header;
2676
2677         cqp = qp->pd->dev->cqp;
2678         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2679         if (!wqe)
2680                 return I40IW_ERR_RING_FULL;
2681
2682         temp = info->ae_code |
2683                LS_64(info->ae_source, I40IW_CQPSQ_FWQE_AESOURCE);
2684
2685         set_64bit_val(wqe, 8, temp);
2686
2687         header = qp->qp_uk.qp_id |
2688                  LS_64(I40IW_CQP_OP_GEN_AE, I40IW_CQPSQ_OPCODE) |
2689                  LS_64(1, I40IW_CQPSQ_FWQE_GENERATE_AE) |
2690                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2691
2692         i40iw_insert_wqe_hdr(wqe, header);
2693
2694         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "GEN_AE WQE",
2695                         wqe, I40IW_CQP_WQE_SIZE * 8);
2696
2697         if (post_sq)
2698                 i40iw_sc_cqp_post_sq(cqp);
2699         return 0;
2700 }
2701
2702 /**
2703  * i40iw_sc_qp_upload_context - upload qp's context
2704  * @dev: sc device struct
2705  * @info: upload context info ptr for return
2706  * @scratch: u64 saved to be used during cqp completion
2707  * @post_sq: flag for cqp db to ring
2708  */
2709 static enum i40iw_status_code i40iw_sc_qp_upload_context(
2710                                         struct i40iw_sc_dev *dev,
2711                                         struct i40iw_upload_context_info *info,
2712                                         u64 scratch,
2713                                         bool post_sq)
2714 {
2715         u64 *wqe;
2716         struct i40iw_sc_cqp *cqp;
2717         u64 header;
2718
2719         cqp = dev->cqp;
2720         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2721         if (!wqe)
2722                 return I40IW_ERR_RING_FULL;
2723         set_64bit_val(wqe, 16, info->buf_pa);
2724
2725         header = LS_64(info->qp_id, I40IW_CQPSQ_UCTX_QPID) |
2726                  LS_64(I40IW_CQP_OP_UPLOAD_CONTEXT, I40IW_CQPSQ_OPCODE) |
2727                  LS_64(info->qp_type, I40IW_CQPSQ_UCTX_QPTYPE) |
2728                  LS_64(info->raw_format, I40IW_CQPSQ_UCTX_RAWFORMAT) |
2729                  LS_64(info->freeze_qp, I40IW_CQPSQ_UCTX_FREEZEQP) |
2730                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2731
2732         i40iw_insert_wqe_hdr(wqe, header);
2733
2734         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QP_UPLOAD_CTX WQE",
2735                         wqe, I40IW_CQP_WQE_SIZE * 8);
2736
2737         if (post_sq)
2738                 i40iw_sc_cqp_post_sq(cqp);
2739         return 0;
2740 }
2741
2742 /**
2743  * i40iw_sc_qp_setctx - set qp's context
2744  * @qp: sc qp
2745  * @qp_ctx: context ptr
2746  * @info: ctx info
2747  */
2748 static enum i40iw_status_code i40iw_sc_qp_setctx(
2749                                 struct i40iw_sc_qp *qp,
2750                                 u64 *qp_ctx,
2751                                 struct i40iw_qp_host_ctx_info *info)
2752 {
2753         struct i40iwarp_offload_info *iw;
2754         struct i40iw_tcp_offload_info *tcp;
2755         struct i40iw_sc_vsi *vsi;
2756         struct i40iw_sc_dev *dev;
2757         u64 qw0, qw3, qw7 = 0;
2758
2759         iw = info->iwarp_info;
2760         tcp = info->tcp_info;
2761         vsi = qp->vsi;
2762         dev = qp->dev;
2763         if (info->add_to_qoslist) {
2764                 qp->user_pri = info->user_pri;
2765                 i40iw_qp_add_qos(qp);
2766                 i40iw_debug(qp->dev, I40IW_DEBUG_DCB, "%s qp[%d] UP[%d] qset[%d]\n",
2767                             __func__, qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle);
2768         }
2769         qw0 = LS_64(qp->qp_uk.rq_wqe_size, I40IWQPC_RQWQESIZE) |
2770               LS_64(info->err_rq_idx_valid, I40IWQPC_ERR_RQ_IDX_VALID) |
2771               LS_64(qp->rcv_tph_en, I40IWQPC_RCVTPHEN) |
2772               LS_64(qp->xmit_tph_en, I40IWQPC_XMITTPHEN) |
2773               LS_64(qp->rq_tph_en, I40IWQPC_RQTPHEN) |
2774               LS_64(qp->sq_tph_en, I40IWQPC_SQTPHEN) |
2775               LS_64(info->push_idx, I40IWQPC_PPIDX) |
2776               LS_64(info->push_mode_en, I40IWQPC_PMENA);
2777
2778         set_64bit_val(qp_ctx, 8, qp->sq_pa);
2779         set_64bit_val(qp_ctx, 16, qp->rq_pa);
2780
2781         qw3 = LS_64(qp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) |
2782               LS_64(qp->hw_rq_size, I40IWQPC_RQSIZE) |
2783               LS_64(qp->hw_sq_size, I40IWQPC_SQSIZE);
2784
2785         set_64bit_val(qp_ctx,
2786                       128,
2787                       LS_64(info->err_rq_idx, I40IWQPC_ERR_RQ_IDX));
2788
2789         set_64bit_val(qp_ctx,
2790                       136,
2791                       LS_64(info->send_cq_num, I40IWQPC_TXCQNUM) |
2792                       LS_64(info->rcv_cq_num, I40IWQPC_RXCQNUM));
2793
2794         set_64bit_val(qp_ctx,
2795                       168,
2796                       LS_64(info->qp_compl_ctx, I40IWQPC_QPCOMPCTX));
2797         set_64bit_val(qp_ctx,
2798                       176,
2799                       LS_64(qp->sq_tph_val, I40IWQPC_SQTPHVAL) |
2800                       LS_64(qp->rq_tph_val, I40IWQPC_RQTPHVAL) |
2801                       LS_64(qp->qs_handle, I40IWQPC_QSHANDLE) |
2802                       LS_64(vsi->exception_lan_queue, I40IWQPC_EXCEPTION_LAN_QUEUE));
2803
2804         if (info->iwarp_info_valid) {
2805                 qw0 |= LS_64(iw->ddp_ver, I40IWQPC_DDP_VER) |
2806                        LS_64(iw->rdmap_ver, I40IWQPC_RDMAP_VER);
2807
2808                 qw7 |= LS_64(iw->pd_id, I40IWQPC_PDIDX);
2809                 set_64bit_val(qp_ctx,
2810                               144,
2811                               LS_64(qp->q2_pa, I40IWQPC_Q2ADDR) |
2812                               LS_64(vsi->fcn_id, I40IWQPC_STAT_INDEX));
2813                 set_64bit_val(qp_ctx,
2814                               152,
2815                               LS_64(iw->last_byte_sent, I40IWQPC_LASTBYTESENT));
2816
2817                 set_64bit_val(qp_ctx,
2818                               160,
2819                               LS_64(iw->ord_size, I40IWQPC_ORDSIZE) |
2820                               LS_64(iw->ird_size, I40IWQPC_IRDSIZE) |
2821                               LS_64(iw->wr_rdresp_en, I40IWQPC_WRRDRSPOK) |
2822                               LS_64(iw->rd_enable, I40IWQPC_RDOK) |
2823                               LS_64(iw->snd_mark_en, I40IWQPC_SNDMARKERS) |
2824                               LS_64(iw->bind_en, I40IWQPC_BINDEN) |
2825                               LS_64(iw->fast_reg_en, I40IWQPC_FASTREGEN) |
2826                               LS_64(iw->priv_mode_en, I40IWQPC_PRIVEN) |
2827                               LS_64((((vsi->stats_fcn_id_alloc) &&
2828                                       (dev->is_pf) && (vsi->fcn_id >= I40IW_FIRST_NON_PF_STAT)) ? 1 : 0),
2829                                     I40IWQPC_USESTATSINSTANCE) |
2830                               LS_64(1, I40IWQPC_IWARPMODE) |
2831                               LS_64(iw->rcv_mark_en, I40IWQPC_RCVMARKERS) |
2832                               LS_64(iw->align_hdrs, I40IWQPC_ALIGNHDRS) |
2833                               LS_64(iw->rcv_no_mpa_crc, I40IWQPC_RCVNOMPACRC) |
2834                               LS_64(iw->rcv_mark_offset, I40IWQPC_RCVMARKOFFSET) |
2835                               LS_64(iw->snd_mark_offset, I40IWQPC_SNDMARKOFFSET));
2836         }
2837         if (info->tcp_info_valid) {
2838                 qw0 |= LS_64(tcp->ipv4, I40IWQPC_IPV4) |
2839                        LS_64(tcp->no_nagle, I40IWQPC_NONAGLE) |
2840                        LS_64(tcp->insert_vlan_tag, I40IWQPC_INSERTVLANTAG) |
2841                        LS_64(tcp->time_stamp, I40IWQPC_TIMESTAMP) |
2842                        LS_64(tcp->cwnd_inc_limit, I40IWQPC_LIMIT) |
2843                        LS_64(tcp->drop_ooo_seg, I40IWQPC_DROPOOOSEG) |
2844                        LS_64(tcp->dup_ack_thresh, I40IWQPC_DUPACK_THRESH);
2845
2846                 qw3 |= LS_64(tcp->ttl, I40IWQPC_TTL) |
2847                        LS_64(tcp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) |
2848                        LS_64(tcp->avoid_stretch_ack, I40IWQPC_AVOIDSTRETCHACK) |
2849                        LS_64(tcp->tos, I40IWQPC_TOS) |
2850                        LS_64(tcp->src_port, I40IWQPC_SRCPORTNUM) |
2851                        LS_64(tcp->dst_port, I40IWQPC_DESTPORTNUM);
2852
2853                 qp->src_mac_addr_idx = tcp->src_mac_addr_idx;
2854                 set_64bit_val(qp_ctx,
2855                               32,
2856                               LS_64(tcp->dest_ip_addr2, I40IWQPC_DESTIPADDR2) |
2857                               LS_64(tcp->dest_ip_addr3, I40IWQPC_DESTIPADDR3));
2858
2859                 set_64bit_val(qp_ctx,
2860                               40,
2861                               LS_64(tcp->dest_ip_addr0, I40IWQPC_DESTIPADDR0) |
2862                               LS_64(tcp->dest_ip_addr1, I40IWQPC_DESTIPADDR1));
2863
2864                 set_64bit_val(qp_ctx,
2865                               48,
2866                               LS_64(tcp->snd_mss, I40IWQPC_SNDMSS) |
2867                                 LS_64(tcp->vlan_tag, I40IWQPC_VLANTAG) |
2868                                 LS_64(tcp->arp_idx, I40IWQPC_ARPIDX));
2869
2870                 qw7 |= LS_64(tcp->flow_label, I40IWQPC_FLOWLABEL) |
2871                        LS_64(tcp->wscale, I40IWQPC_WSCALE) |
2872                        LS_64(tcp->ignore_tcp_opt, I40IWQPC_IGNORE_TCP_OPT) |
2873                        LS_64(tcp->ignore_tcp_uns_opt, I40IWQPC_IGNORE_TCP_UNS_OPT) |
2874                        LS_64(tcp->tcp_state, I40IWQPC_TCPSTATE) |
2875                        LS_64(tcp->rcv_wscale, I40IWQPC_RCVSCALE) |
2876                        LS_64(tcp->snd_wscale, I40IWQPC_SNDSCALE);
2877
2878                 set_64bit_val(qp_ctx,
2879                               72,
2880                               LS_64(tcp->time_stamp_recent, I40IWQPC_TIMESTAMP_RECENT) |
2881                               LS_64(tcp->time_stamp_age, I40IWQPC_TIMESTAMP_AGE));
2882                 set_64bit_val(qp_ctx,
2883                               80,
2884                               LS_64(tcp->snd_nxt, I40IWQPC_SNDNXT) |
2885                               LS_64(tcp->snd_wnd, I40IWQPC_SNDWND));
2886
2887                 set_64bit_val(qp_ctx,
2888                               88,
2889                               LS_64(tcp->rcv_nxt, I40IWQPC_RCVNXT) |
2890                               LS_64(tcp->rcv_wnd, I40IWQPC_RCVWND));
2891                 set_64bit_val(qp_ctx,
2892                               96,
2893                               LS_64(tcp->snd_max, I40IWQPC_SNDMAX) |
2894                               LS_64(tcp->snd_una, I40IWQPC_SNDUNA));
2895                 set_64bit_val(qp_ctx,
2896                               104,
2897                               LS_64(tcp->srtt, I40IWQPC_SRTT) |
2898                               LS_64(tcp->rtt_var, I40IWQPC_RTTVAR));
2899                 set_64bit_val(qp_ctx,
2900                               112,
2901                               LS_64(tcp->ss_thresh, I40IWQPC_SSTHRESH) |
2902                               LS_64(tcp->cwnd, I40IWQPC_CWND));
2903                 set_64bit_val(qp_ctx,
2904                               120,
2905                               LS_64(tcp->snd_wl1, I40IWQPC_SNDWL1) |
2906                               LS_64(tcp->snd_wl2, I40IWQPC_SNDWL2));
2907                 set_64bit_val(qp_ctx,
2908                               128,
2909                               LS_64(tcp->max_snd_window, I40IWQPC_MAXSNDWND) |
2910                               LS_64(tcp->rexmit_thresh, I40IWQPC_REXMIT_THRESH));
2911                 set_64bit_val(qp_ctx,
2912                               184,
2913                               LS_64(tcp->local_ipaddr3, I40IWQPC_LOCAL_IPADDR3) |
2914                               LS_64(tcp->local_ipaddr2, I40IWQPC_LOCAL_IPADDR2));
2915                 set_64bit_val(qp_ctx,
2916                               192,
2917                               LS_64(tcp->local_ipaddr1, I40IWQPC_LOCAL_IPADDR1) |
2918                               LS_64(tcp->local_ipaddr0, I40IWQPC_LOCAL_IPADDR0));
2919         }
2920
2921         set_64bit_val(qp_ctx, 0, qw0);
2922         set_64bit_val(qp_ctx, 24, qw3);
2923         set_64bit_val(qp_ctx, 56, qw7);
2924
2925         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "QP_HOST)CTX WQE",
2926                         qp_ctx, I40IW_QP_CTX_SIZE);
2927         return 0;
2928 }
2929
2930 /**
2931  * i40iw_sc_alloc_stag - mr stag alloc
2932  * @dev: sc device struct
2933  * @info: stag info
2934  * @scratch: u64 saved to be used during cqp completion
2935  * @post_sq: flag for cqp db to ring
2936  */
2937 static enum i40iw_status_code i40iw_sc_alloc_stag(
2938                                 struct i40iw_sc_dev *dev,
2939                                 struct i40iw_allocate_stag_info *info,
2940                                 u64 scratch,
2941                                 bool post_sq)
2942 {
2943         u64 *wqe;
2944         struct i40iw_sc_cqp *cqp;
2945         u64 header;
2946         enum i40iw_page_size page_size;
2947
2948         page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
2949         cqp = dev->cqp;
2950         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2951         if (!wqe)
2952                 return I40IW_ERR_RING_FULL;
2953         set_64bit_val(wqe,
2954                       8,
2955                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID) |
2956                       LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN));
2957         set_64bit_val(wqe,
2958                       16,
2959                       LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
2960         set_64bit_val(wqe,
2961                       40,
2962                       LS_64(info->hmc_fcn_index, I40IW_CQPSQ_STAG_HMCFNIDX));
2963
2964         header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) |
2965                  LS_64(1, I40IW_CQPSQ_STAG_MR) |
2966                  LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
2967                  LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |
2968                  LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
2969                  LS_64(info->remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
2970                  LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) |
2971                  LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) |
2972                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2973
2974         i40iw_insert_wqe_hdr(wqe, header);
2975
2976         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "ALLOC_STAG WQE",
2977                         wqe, I40IW_CQP_WQE_SIZE * 8);
2978
2979         if (post_sq)
2980                 i40iw_sc_cqp_post_sq(cqp);
2981         return 0;
2982 }
2983
2984 /**
2985  * i40iw_sc_mr_reg_non_shared - non-shared mr registration
2986  * @dev: sc device struct
2987  * @info: mr info
2988  * @scratch: u64 saved to be used during cqp completion
2989  * @post_sq: flag for cqp db to ring
2990  */
2991 static enum i40iw_status_code i40iw_sc_mr_reg_non_shared(
2992                                 struct i40iw_sc_dev *dev,
2993                                 struct i40iw_reg_ns_stag_info *info,
2994                                 u64 scratch,
2995                                 bool post_sq)
2996 {
2997         u64 *wqe;
2998         u64 temp;
2999         struct i40iw_sc_cqp *cqp;
3000         u64 header;
3001         u32 pble_obj_cnt;
3002         bool remote_access;
3003         u8 addr_type;
3004         enum i40iw_page_size page_size;
3005
3006         page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
3007         if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY |
3008                                    I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))
3009                 remote_access = true;
3010         else
3011                 remote_access = false;
3012
3013         pble_obj_cnt = dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
3014
3015         if (info->chunk_size && (info->first_pm_pbl_index >= pble_obj_cnt))
3016                 return I40IW_ERR_INVALID_PBLE_INDEX;
3017
3018         cqp = dev->cqp;
3019         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3020         if (!wqe)
3021                 return I40IW_ERR_RING_FULL;
3022
3023         temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo;
3024         set_64bit_val(wqe, 0, temp);
3025
3026         set_64bit_val(wqe,
3027                       8,
3028                       LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN) |
3029                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
3030
3031         set_64bit_val(wqe,
3032                       16,
3033                       LS_64(info->stag_key, I40IW_CQPSQ_STAG_KEY) |
3034                       LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
3035         if (!info->chunk_size) {
3036                 set_64bit_val(wqe, 32, info->reg_addr_pa);
3037                 set_64bit_val(wqe, 48, 0);
3038         } else {
3039                 set_64bit_val(wqe, 32, 0);
3040                 set_64bit_val(wqe, 48, info->first_pm_pbl_index);
3041         }
3042         set_64bit_val(wqe, 40, info->hmc_fcn_index);
3043         set_64bit_val(wqe, 56, 0);
3044
3045         addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;
3046         header = LS_64(I40IW_CQP_OP_REG_MR, I40IW_CQPSQ_OPCODE) |
3047                  LS_64(1, I40IW_CQPSQ_STAG_MR) |
3048                  LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |
3049                  LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
3050                  LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
3051                  LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
3052                  LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) |
3053                  LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) |
3054                  LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) |
3055                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3056
3057         i40iw_insert_wqe_hdr(wqe, header);
3058
3059         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_NS WQE",
3060                         wqe, I40IW_CQP_WQE_SIZE * 8);
3061
3062         if (post_sq)
3063                 i40iw_sc_cqp_post_sq(cqp);
3064         return 0;
3065 }
3066
3067 /**
3068  * i40iw_sc_mr_reg_shared - registered shared memory region
3069  * @dev: sc device struct
3070  * @info: info for shared memory registeration
3071  * @scratch: u64 saved to be used during cqp completion
3072  * @post_sq: flag for cqp db to ring
3073  */
3074 static enum i40iw_status_code i40iw_sc_mr_reg_shared(
3075                                         struct i40iw_sc_dev *dev,
3076                                         struct i40iw_register_shared_stag *info,
3077                                         u64 scratch,
3078                                         bool post_sq)
3079 {
3080         u64 *wqe;
3081         struct i40iw_sc_cqp *cqp;
3082         u64 temp, va64, fbo, header;
3083         u32 va32;
3084         bool remote_access;
3085         u8 addr_type;
3086
3087         if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY |
3088                                    I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))
3089                 remote_access = true;
3090         else
3091                 remote_access = false;
3092         cqp = dev->cqp;
3093         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3094         if (!wqe)
3095                 return I40IW_ERR_RING_FULL;
3096         va64 = (uintptr_t)(info->va);
3097         va32 = (u32)(va64 & 0x00000000FFFFFFFF);
3098         fbo = (u64)(va32 & (4096 - 1));
3099
3100         set_64bit_val(wqe,
3101                       0,
3102                       (info->addr_type == I40IW_ADDR_TYPE_VA_BASED ? (uintptr_t)info->va : fbo));
3103
3104         set_64bit_val(wqe,
3105                       8,
3106                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
3107         temp = LS_64(info->new_stag_key, I40IW_CQPSQ_STAG_KEY) |
3108                LS_64(info->new_stag_idx, I40IW_CQPSQ_STAG_IDX) |
3109                LS_64(info->parent_stag_idx, I40IW_CQPSQ_STAG_PARENTSTAGIDX);
3110         set_64bit_val(wqe, 16, temp);
3111
3112         addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;
3113         header = LS_64(I40IW_CQP_OP_REG_SMR, I40IW_CQPSQ_OPCODE) |
3114                  LS_64(1, I40IW_CQPSQ_STAG_MR) |
3115                  LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
3116                  LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
3117                  LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) |
3118                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3119
3120         i40iw_insert_wqe_hdr(wqe, header);
3121
3122         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_SHARED WQE",
3123                         wqe, I40IW_CQP_WQE_SIZE * 8);
3124
3125         if (post_sq)
3126                 i40iw_sc_cqp_post_sq(cqp);
3127         return 0;
3128 }
3129
3130 /**
3131  * i40iw_sc_dealloc_stag - deallocate stag
3132  * @dev: sc device struct
3133  * @info: dealloc stag info
3134  * @scratch: u64 saved to be used during cqp completion
3135  * @post_sq: flag for cqp db to ring
3136  */
3137 static enum i40iw_status_code i40iw_sc_dealloc_stag(
3138                                         struct i40iw_sc_dev *dev,
3139                                         struct i40iw_dealloc_stag_info *info,
3140                                         u64 scratch,
3141                                         bool post_sq)
3142 {
3143         u64 header;
3144         u64 *wqe;
3145         struct i40iw_sc_cqp *cqp;
3146
3147         cqp = dev->cqp;
3148         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3149         if (!wqe)
3150                 return I40IW_ERR_RING_FULL;
3151         set_64bit_val(wqe,
3152                       8,
3153                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
3154         set_64bit_val(wqe,
3155                       16,
3156                       LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
3157
3158         header = LS_64(I40IW_CQP_OP_DEALLOC_STAG, I40IW_CQPSQ_OPCODE) |
3159                  LS_64(info->mr, I40IW_CQPSQ_STAG_MR) |
3160                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3161
3162         i40iw_insert_wqe_hdr(wqe, header);
3163
3164         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "DEALLOC_STAG WQE",
3165                         wqe, I40IW_CQP_WQE_SIZE * 8);
3166
3167         if (post_sq)
3168                 i40iw_sc_cqp_post_sq(cqp);
3169         return 0;
3170 }
3171
3172 /**
3173  * i40iw_sc_query_stag - query hardware for stag
3174  * @dev: sc device struct
3175  * @scratch: u64 saved to be used during cqp completion
3176  * @stag_index: stag index for query
3177  * @post_sq: flag for cqp db to ring
3178  */
3179 static enum i40iw_status_code i40iw_sc_query_stag(struct i40iw_sc_dev *dev,
3180                                                   u64 scratch,
3181                                                   u32 stag_index,
3182                                                   bool post_sq)
3183 {
3184         u64 header;
3185         u64 *wqe;
3186         struct i40iw_sc_cqp *cqp;
3187
3188         cqp = dev->cqp;
3189         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3190         if (!wqe)
3191                 return I40IW_ERR_RING_FULL;
3192         set_64bit_val(wqe,
3193                       16,
3194                       LS_64(stag_index, I40IW_CQPSQ_QUERYSTAG_IDX));
3195
3196         header = LS_64(I40IW_CQP_OP_QUERY_STAG, I40IW_CQPSQ_OPCODE) |
3197                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3198
3199         i40iw_insert_wqe_hdr(wqe, header);
3200
3201         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QUERY_STAG WQE",
3202                         wqe, I40IW_CQP_WQE_SIZE * 8);
3203
3204         if (post_sq)
3205                 i40iw_sc_cqp_post_sq(cqp);
3206         return 0;
3207 }
3208
3209 /**
3210  * i40iw_sc_mw_alloc - mw allocate
3211  * @dev: sc device struct
3212  * @scratch: u64 saved to be used during cqp completion
3213  * @mw_stag_index:stag index
3214  * @pd_id: pd is for this mw
3215  * @post_sq: flag for cqp db to ring
3216  */
3217 static enum i40iw_status_code i40iw_sc_mw_alloc(
3218                                         struct i40iw_sc_dev *dev,
3219                                         u64 scratch,
3220                                         u32 mw_stag_index,
3221                                         u16 pd_id,
3222                                         bool post_sq)
3223 {
3224         u64 header;
3225         struct i40iw_sc_cqp *cqp;
3226         u64 *wqe;
3227
3228         cqp = dev->cqp;
3229         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3230         if (!wqe)
3231                 return I40IW_ERR_RING_FULL;
3232         set_64bit_val(wqe, 8, LS_64(pd_id, I40IW_CQPSQ_STAG_PDID));
3233         set_64bit_val(wqe,
3234                       16,
3235                       LS_64(mw_stag_index, I40IW_CQPSQ_STAG_IDX));
3236
3237         header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) |
3238                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3239
3240         i40iw_insert_wqe_hdr(wqe, header);
3241
3242         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MW_ALLOC WQE",
3243                         wqe, I40IW_CQP_WQE_SIZE * 8);
3244
3245         if (post_sq)
3246                 i40iw_sc_cqp_post_sq(cqp);
3247         return 0;
3248 }
3249
3250 /**
3251  * i40iw_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
3252  * @qp: sc qp struct
3253  * @info: fast mr info
3254  * @post_sq: flag for cqp db to ring
3255  */
3256 enum i40iw_status_code i40iw_sc_mr_fast_register(
3257                                 struct i40iw_sc_qp *qp,
3258                                 struct i40iw_fast_reg_stag_info *info,
3259                                 bool post_sq)
3260 {
3261         u64 temp, header;
3262         u64 *wqe;
3263         u32 wqe_idx;
3264         enum i40iw_page_size page_size;
3265
3266         page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
3267         wqe = i40iw_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, I40IW_QP_WQE_MIN_SIZE,
3268                                          0, info->wr_id);
3269         if (!wqe)
3270                 return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
3271
3272         i40iw_debug(qp->dev, I40IW_DEBUG_MR, "%s: wr_id[%llxh] wqe_idx[%04d] location[%p]\n",
3273                     __func__, info->wr_id, wqe_idx,
3274                     &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
3275         temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo;
3276         set_64bit_val(wqe, 0, temp);
3277
3278         temp = RS_64(info->first_pm_pbl_index >> 16, I40IWQPSQ_FIRSTPMPBLIDXHI);
3279         set_64bit_val(wqe,
3280                       8,
3281                       LS_64(temp, I40IWQPSQ_FIRSTPMPBLIDXHI) |
3282                       LS_64(info->reg_addr_pa >> I40IWQPSQ_PBLADDR_SHIFT, I40IWQPSQ_PBLADDR));
3283
3284         set_64bit_val(wqe,
3285                       16,
3286                       info->total_len |
3287                       LS_64(info->first_pm_pbl_index, I40IWQPSQ_FIRSTPMPBLIDXLO));
3288
3289         header = LS_64(info->stag_key, I40IWQPSQ_STAGKEY) |
3290                  LS_64(info->stag_idx, I40IWQPSQ_STAGINDEX) |
3291                  LS_64(I40IWQP_OP_FAST_REGISTER, I40IWQPSQ_OPCODE) |
3292                  LS_64(info->chunk_size, I40IWQPSQ_LPBLSIZE) |
3293                  LS_64(page_size, I40IWQPSQ_HPAGESIZE) |
3294                  LS_64(info->access_rights, I40IWQPSQ_STAGRIGHTS) |
3295                  LS_64(info->addr_type, I40IWQPSQ_VABASEDTO) |
3296                  LS_64(info->read_fence, I40IWQPSQ_READFENCE) |
3297                  LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
3298                  LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
3299                  LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3300
3301         i40iw_insert_wqe_hdr(wqe, header);
3302
3303         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "FAST_REG WQE",
3304                         wqe, I40IW_QP_WQE_MIN_SIZE);
3305
3306         if (post_sq)
3307                 i40iw_qp_post_wr(&qp->qp_uk);
3308         return 0;
3309 }
3310
3311 /**
3312  * i40iw_sc_send_lsmm - send last streaming mode message
3313  * @qp: sc qp struct
3314  * @lsmm_buf: buffer with lsmm message
3315  * @size: size of lsmm buffer
3316  * @stag: stag of lsmm buffer
3317  */
3318 static void i40iw_sc_send_lsmm(struct i40iw_sc_qp *qp,
3319                                void *lsmm_buf,
3320                                u32 size,
3321                                i40iw_stag stag)
3322 {
3323         u64 *wqe;
3324         u64 header;
3325         struct i40iw_qp_uk *qp_uk;
3326
3327         qp_uk = &qp->qp_uk;
3328         wqe = qp_uk->sq_base->elem;
3329
3330         set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
3331
3332         set_64bit_val(wqe, 8, (size | LS_64(stag, I40IWQPSQ_FRAG_STAG)));
3333
3334         set_64bit_val(wqe, 16, 0);
3335
3336         header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
3337                  LS_64(1, I40IWQPSQ_STREAMMODE) |
3338                  LS_64(1, I40IWQPSQ_WAITFORRCVPDU) |
3339                  LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3340
3341         i40iw_insert_wqe_hdr(wqe, header);
3342
3343         i40iw_debug_buf(qp->dev, I40IW_DEBUG_QP, "SEND_LSMM WQE",
3344                         wqe, I40IW_QP_WQE_MIN_SIZE);
3345 }
3346
3347 /**
3348  * i40iw_sc_send_lsmm_nostag - for privilege qp
3349  * @qp: sc qp struct
3350  * @lsmm_buf: buffer with lsmm message
3351  * @size: size of lsmm buffer
3352  */
3353 static void i40iw_sc_send_lsmm_nostag(struct i40iw_sc_qp *qp,
3354                                       void *lsmm_buf,
3355                                       u32 size)
3356 {
3357         u64 *wqe;
3358         u64 header;
3359         struct i40iw_qp_uk *qp_uk;
3360
3361         qp_uk = &qp->qp_uk;
3362         wqe = qp_uk->sq_base->elem;
3363
3364         set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
3365
3366         set_64bit_val(wqe, 8, size);
3367
3368         set_64bit_val(wqe, 16, 0);
3369
3370         header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
3371                  LS_64(1, I40IWQPSQ_STREAMMODE) |
3372                  LS_64(1, I40IWQPSQ_WAITFORRCVPDU) |
3373                  LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3374
3375         i40iw_insert_wqe_hdr(wqe, header);
3376
3377         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE",
3378                         wqe, I40IW_QP_WQE_MIN_SIZE);
3379 }
3380
3381 /**
3382  * i40iw_sc_send_rtt - send last read0 or write0
3383  * @qp: sc qp struct
3384  * @read: Do read0 or write0
3385  */
3386 static void i40iw_sc_send_rtt(struct i40iw_sc_qp *qp, bool read)
3387 {
3388         u64 *wqe;
3389         u64 header;
3390         struct i40iw_qp_uk *qp_uk;
3391
3392         qp_uk = &qp->qp_uk;
3393         wqe = qp_uk->sq_base->elem;
3394
3395         set_64bit_val(wqe, 0, 0);
3396         set_64bit_val(wqe, 8, 0);
3397         set_64bit_val(wqe, 16, 0);
3398         if (read) {
3399                 header = LS_64(0x1234, I40IWQPSQ_REMSTAG) |
3400                          LS_64(I40IWQP_OP_RDMA_READ, I40IWQPSQ_OPCODE) |
3401                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3402                 set_64bit_val(wqe, 8, ((u64)0xabcd << 32));
3403         } else {
3404                 header = LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) |
3405                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3406         }
3407
3408         i40iw_insert_wqe_hdr(wqe, header);
3409
3410         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "RTR WQE",
3411                         wqe, I40IW_QP_WQE_MIN_SIZE);
3412 }
3413
3414 /**
3415  * i40iw_sc_post_wqe0 - send wqe with opcode
3416  * @qp: sc qp struct
3417  * @opcode: opcode to use for wqe0
3418  */
3419 static enum i40iw_status_code i40iw_sc_post_wqe0(struct i40iw_sc_qp *qp, u8 opcode)
3420 {
3421         u64 *wqe;
3422         u64 header;
3423         struct i40iw_qp_uk *qp_uk;
3424
3425         qp_uk = &qp->qp_uk;
3426         wqe = qp_uk->sq_base->elem;
3427
3428         if (!wqe)
3429                 return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
3430         switch (opcode) {
3431         case I40IWQP_OP_NOP:
3432                 set_64bit_val(wqe, 0, 0);
3433                 set_64bit_val(wqe, 8, 0);
3434                 set_64bit_val(wqe, 16, 0);
3435                 header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) |
3436                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3437
3438                 i40iw_insert_wqe_hdr(wqe, header);
3439                 break;
3440         case I40IWQP_OP_RDMA_SEND:
3441                 set_64bit_val(wqe, 0, 0);
3442                 set_64bit_val(wqe, 8, 0);
3443                 set_64bit_val(wqe, 16, 0);
3444                 header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
3445                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID) |
3446                          LS_64(1, I40IWQPSQ_STREAMMODE) |
3447                          LS_64(1, I40IWQPSQ_WAITFORRCVPDU);
3448
3449                 i40iw_insert_wqe_hdr(wqe, header);
3450                 break;
3451         default:
3452                 i40iw_debug(qp->dev, I40IW_DEBUG_QP, "%s: Invalid WQE zero opcode\n",
3453                             __func__);
3454                 break;
3455         }
3456         return 0;
3457 }
3458
3459 /**
3460  * i40iw_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
3461  * @dev : ptr to i40iw_dev struct
3462  * @hmc_fn_id: hmc function id
3463  */
3464 enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev, u8 hmc_fn_id)
3465 {
3466         struct i40iw_hmc_info *hmc_info;
3467         struct i40iw_dma_mem query_fpm_mem;
3468         struct i40iw_virt_mem virt_mem;
3469         struct i40iw_vfdev *vf_dev = NULL;
3470         u32 mem_size;
3471         enum i40iw_status_code ret_code = 0;
3472         bool poll_registers = true;
3473         u16 iw_vf_idx;
3474         u8 wait_type;
3475
3476         if (hmc_fn_id >= I40IW_MAX_VF_FPM_ID ||
3477             (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID))
3478                 return I40IW_ERR_INVALID_HMCFN_ID;
3479
3480         i40iw_debug(dev, I40IW_DEBUG_HMC, "hmc_fn_id %u, dev->hmc_fn_id %u\n", hmc_fn_id,
3481                     dev->hmc_fn_id);
3482         if (hmc_fn_id == dev->hmc_fn_id) {
3483                 hmc_info = dev->hmc_info;
3484                 query_fpm_mem.pa = dev->fpm_query_buf_pa;
3485                 query_fpm_mem.va = dev->fpm_query_buf;
3486         } else {
3487                 vf_dev = i40iw_vfdev_from_fpm(dev, hmc_fn_id);
3488                 if (!vf_dev)
3489                         return I40IW_ERR_INVALID_VF_ID;
3490
3491                 hmc_info = &vf_dev->hmc_info;
3492                 iw_vf_idx = vf_dev->iw_vf_idx;
3493                 i40iw_debug(dev, I40IW_DEBUG_HMC, "vf_dev %p, hmc_info %p, hmc_obj %p\n", vf_dev,
3494                             hmc_info, hmc_info->hmc_obj);
3495                 if (!vf_dev->fpm_query_buf) {
3496                         if (!dev->vf_fpm_query_buf[iw_vf_idx].va) {
3497                                 ret_code = i40iw_alloc_query_fpm_buf(dev,
3498                                                                      &dev->vf_fpm_query_buf[iw_vf_idx]);
3499                                 if (ret_code)
3500                                         return ret_code;
3501                         }
3502                         vf_dev->fpm_query_buf = dev->vf_fpm_query_buf[iw_vf_idx].va;
3503                         vf_dev->fpm_query_buf_pa = dev->vf_fpm_query_buf[iw_vf_idx].pa;
3504                 }
3505                 query_fpm_mem.pa = vf_dev->fpm_query_buf_pa;
3506                 query_fpm_mem.va = vf_dev->fpm_query_buf;
3507                 /**
3508                  * It is HARDWARE specific:
3509                  * this call is done by PF for VF and
3510                  * i40iw_sc_query_fpm_values needs ccq poll
3511                  * because PF ccq is already created.
3512                  */
3513                 poll_registers = false;
3514         }
3515
3516         hmc_info->hmc_fn_id = hmc_fn_id;
3517
3518         if (hmc_fn_id != dev->hmc_fn_id) {
3519                 ret_code =
3520                         i40iw_cqp_query_fpm_values_cmd(dev, &query_fpm_mem, hmc_fn_id);
3521         } else {
3522                 wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :
3523                             (u8)I40IW_CQP_WAIT_POLL_CQ;
3524
3525                 ret_code = i40iw_sc_query_fpm_values(
3526                                         dev->cqp,
3527                                         0,
3528                                         hmc_info->hmc_fn_id,
3529                                         &query_fpm_mem,
3530                                         true,
3531                                         wait_type);
3532         }
3533         if (ret_code)
3534                 return ret_code;
3535
3536         /* parse the fpm_query_buf and fill hmc obj info */
3537         ret_code =
3538                 i40iw_sc_parse_fpm_query_buf((u64 *)query_fpm_mem.va,
3539                                              hmc_info,
3540                                              &dev->hmc_fpm_misc);
3541         if (ret_code)
3542                 return ret_code;
3543         i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "QUERY FPM BUFFER",
3544                         query_fpm_mem.va, I40IW_QUERY_FPM_BUF_SIZE);
3545
3546         if (hmc_fn_id != dev->hmc_fn_id) {
3547                 i40iw_cqp_commit_fpm_values_cmd(dev, &query_fpm_mem, hmc_fn_id);
3548
3549                 /* parse the fpm_commit_buf and fill hmc obj info */
3550                 i40iw_sc_parse_fpm_commit_buf((u64 *)query_fpm_mem.va, hmc_info->hmc_obj, &hmc_info->sd_table.sd_cnt);
3551                 mem_size = sizeof(struct i40iw_hmc_sd_entry) *
3552                            (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index);
3553                 ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size);
3554                 if (ret_code)
3555                         return ret_code;
3556                 hmc_info->sd_table.sd_entry = virt_mem.va;
3557         }
3558
3559         return ret_code;
3560 }
3561
3562 /**
3563  * i40iw_sc_configure_iw_fpm() - commits hmc obj cnt values using cqp command and
3564  * populates fpm base address in hmc_info
3565  * @dev : ptr to i40iw_dev struct
3566  * @hmc_fn_id: hmc function id
3567  */
3568 static enum i40iw_status_code i40iw_sc_configure_iw_fpm(struct i40iw_sc_dev *dev,
3569                                                         u8 hmc_fn_id)
3570 {
3571         struct i40iw_hmc_info *hmc_info;
3572         struct i40iw_hmc_obj_info *obj_info;
3573         u64 *buf;
3574         struct i40iw_dma_mem commit_fpm_mem;
3575         u32 i, j;
3576         enum i40iw_status_code ret_code = 0;
3577         bool poll_registers = true;
3578         u8 wait_type;
3579
3580         if (hmc_fn_id >= I40IW_MAX_VF_FPM_ID ||
3581             (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID))
3582                 return I40IW_ERR_INVALID_HMCFN_ID;
3583
3584         if (hmc_fn_id == dev->hmc_fn_id) {
3585                 hmc_info = dev->hmc_info;
3586         } else {
3587                 hmc_info = i40iw_vf_hmcinfo_from_fpm(dev, hmc_fn_id);
3588                 poll_registers = false;
3589         }
3590         if (!hmc_info)
3591                 return I40IW_ERR_BAD_PTR;
3592
3593         obj_info = hmc_info->hmc_obj;
3594         buf = dev->fpm_commit_buf;
3595
3596         /* copy cnt values in commit buf */
3597         for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE;
3598              i++, j += 8)
3599                 set_64bit_val(buf, j, (u64)obj_info[i].cnt);
3600
3601         set_64bit_val(buf, 40, 0);   /* APBVT rsvd */
3602
3603         commit_fpm_mem.pa = dev->fpm_commit_buf_pa;
3604         commit_fpm_mem.va = dev->fpm_commit_buf;
3605         wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :
3606                         (u8)I40IW_CQP_WAIT_POLL_CQ;
3607         ret_code = i40iw_sc_commit_fpm_values(
3608                                         dev->cqp,
3609                                         0,
3610                                         hmc_info->hmc_fn_id,
3611                                         &commit_fpm_mem,
3612                                         true,
3613                                         wait_type);
3614
3615         /* parse the fpm_commit_buf and fill hmc obj info */
3616         if (!ret_code)
3617                 ret_code = i40iw_sc_parse_fpm_commit_buf(dev->fpm_commit_buf,
3618                                                          hmc_info->hmc_obj,
3619                                                          &hmc_info->sd_table.sd_cnt);
3620
3621         i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "COMMIT FPM BUFFER",
3622                         commit_fpm_mem.va, I40IW_COMMIT_FPM_BUF_SIZE);
3623
3624         return ret_code;
3625 }
3626
3627 /**
3628  * cqp_sds_wqe_fill - fill cqp wqe doe sd
3629  * @cqp: struct for cqp hw
3630  * @info; sd info for wqe
3631  * @scratch: u64 saved to be used during cqp completion
3632  */
3633 static enum i40iw_status_code cqp_sds_wqe_fill(struct i40iw_sc_cqp *cqp,
3634                                                struct i40iw_update_sds_info *info,
3635                                                u64 scratch)
3636 {
3637         u64 data;
3638         u64 header;
3639         u64 *wqe;
3640         int mem_entries, wqe_entries;
3641         struct i40iw_dma_mem *sdbuf = &cqp->sdbuf;
3642         u64 offset;
3643         u32 wqe_idx;
3644
3645         wqe = i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
3646         if (!wqe)
3647                 return I40IW_ERR_RING_FULL;
3648
3649         I40IW_CQP_INIT_WQE(wqe);
3650         wqe_entries = (info->cnt > 3) ? 3 : info->cnt;
3651         mem_entries = info->cnt - wqe_entries;
3652
3653         header = LS_64(I40IW_CQP_OP_UPDATE_PE_SDS, I40IW_CQPSQ_OPCODE) |
3654                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
3655                  LS_64(mem_entries, I40IW_CQPSQ_UPESD_ENTRY_COUNT);
3656
3657         if (mem_entries) {
3658                 offset = wqe_idx * I40IW_UPDATE_SD_BUF_SIZE;
3659                 memcpy((char *)sdbuf->va + offset, &info->entry[3],
3660                        mem_entries << 4);
3661                 data = (u64)sdbuf->pa + offset;
3662         } else {
3663                 data = 0;
3664         }
3665         data |= LS_64(info->hmc_fn_id, I40IW_CQPSQ_UPESD_HMCFNID);
3666
3667         set_64bit_val(wqe, 16, data);
3668
3669         switch (wqe_entries) {
3670         case 3:
3671                 set_64bit_val(wqe, 48,
3672                               (LS_64(info->entry[2].cmd, I40IW_CQPSQ_UPESD_SDCMD) |
3673                                         LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));
3674
3675                 set_64bit_val(wqe, 56, info->entry[2].data);
3676                 /* fallthrough */
3677         case 2:
3678                 set_64bit_val(wqe, 32,
3679                               (LS_64(info->entry[1].cmd, I40IW_CQPSQ_UPESD_SDCMD) |
3680                                         LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));
3681
3682                 set_64bit_val(wqe, 40, info->entry[1].data);
3683                 /* fallthrough */
3684         case 1:
3685                 set_64bit_val(wqe, 0,
3686                               LS_64(info->entry[0].cmd, I40IW_CQPSQ_UPESD_SDCMD));
3687
3688                 set_64bit_val(wqe, 8, info->entry[0].data);
3689                 break;
3690         default:
3691                 break;
3692         }
3693
3694         i40iw_insert_wqe_hdr(wqe, header);
3695
3696         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "UPDATE_PE_SDS WQE",
3697                         wqe, I40IW_CQP_WQE_SIZE * 8);
3698         return 0;
3699 }
3700
3701 /**
3702  * i40iw_update_pe_sds - cqp wqe for sd
3703  * @dev: ptr to i40iw_dev struct
3704  * @info: sd info for sd's
3705  * @scratch: u64 saved to be used during cqp completion
3706  */
3707 static enum i40iw_status_code i40iw_update_pe_sds(struct i40iw_sc_dev *dev,
3708                                                   struct i40iw_update_sds_info *info,
3709                                                   u64 scratch)
3710 {
3711         struct i40iw_sc_cqp *cqp = dev->cqp;
3712         enum i40iw_status_code ret_code;
3713
3714         ret_code = cqp_sds_wqe_fill(cqp, info, scratch);
3715         if (!ret_code)
3716                 i40iw_sc_cqp_post_sq(cqp);
3717
3718         return ret_code;
3719 }
3720
3721 /**
3722  * i40iw_update_sds_noccq - update sd before ccq created
3723  * @dev: sc device struct
3724  * @info: sd info for sd's
3725  */
3726 enum i40iw_status_code i40iw_update_sds_noccq(struct i40iw_sc_dev *dev,
3727                                               struct i40iw_update_sds_info *info)
3728 {
3729         u32 error, val, tail;
3730         struct i40iw_sc_cqp *cqp = dev->cqp;
3731         enum i40iw_status_code ret_code;
3732
3733         ret_code = cqp_sds_wqe_fill(cqp, info, 0);
3734         if (ret_code)
3735                 return ret_code;
3736         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
3737         if (error)
3738                 return I40IW_ERR_CQP_COMPL_ERROR;
3739
3740         i40iw_sc_cqp_post_sq(cqp);
3741         ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
3742
3743         return ret_code;
3744 }
3745
3746 /**
3747  * i40iw_sc_suspend_qp - suspend qp for param change
3748  * @cqp: struct for cqp hw
3749  * @qp: sc qp struct
3750  * @scratch: u64 saved to be used during cqp completion
3751  */
3752 enum i40iw_status_code i40iw_sc_suspend_qp(struct i40iw_sc_cqp *cqp,
3753                                            struct i40iw_sc_qp *qp,
3754                                            u64 scratch)
3755 {
3756         u64 header;
3757         u64 *wqe;
3758
3759         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3760         if (!wqe)
3761                 return I40IW_ERR_RING_FULL;
3762         header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_SUSPENDQP_QPID) |
3763                  LS_64(I40IW_CQP_OP_SUSPEND_QP, I40IW_CQPSQ_OPCODE) |
3764                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3765
3766         i40iw_insert_wqe_hdr(wqe, header);
3767
3768         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SUSPEND_QP WQE",
3769                         wqe, I40IW_CQP_WQE_SIZE * 8);
3770
3771         i40iw_sc_cqp_post_sq(cqp);
3772         return 0;
3773 }
3774
3775 /**
3776  * i40iw_sc_resume_qp - resume qp after suspend
3777  * @cqp: struct for cqp hw
3778  * @qp: sc qp struct
3779  * @scratch: u64 saved to be used during cqp completion
3780  */
3781 enum i40iw_status_code i40iw_sc_resume_qp(struct i40iw_sc_cqp *cqp,
3782                                           struct i40iw_sc_qp *qp,
3783                                           u64 scratch)
3784 {
3785         u64 header;
3786         u64 *wqe;
3787
3788         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3789         if (!wqe)
3790                 return I40IW_ERR_RING_FULL;
3791         set_64bit_val(wqe,
3792                       16,
3793                         LS_64(qp->qs_handle, I40IW_CQPSQ_RESUMEQP_QSHANDLE));
3794
3795         header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_RESUMEQP_QPID) |
3796                  LS_64(I40IW_CQP_OP_RESUME_QP, I40IW_CQPSQ_OPCODE) |
3797                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3798
3799         i40iw_insert_wqe_hdr(wqe, header);
3800
3801         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "RESUME_QP WQE",
3802                         wqe, I40IW_CQP_WQE_SIZE * 8);
3803
3804         i40iw_sc_cqp_post_sq(cqp);
3805         return 0;
3806 }
3807
3808 /**
3809  * i40iw_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
3810  * @cqp: struct for cqp hw
3811  * @scratch: u64 saved to be used during cqp completion
3812  * @hmc_fn_id: hmc function id
3813  * @post_sq: flag for cqp db to ring
3814  * @poll_registers: flag to poll register for cqp completion
3815  */
3816 enum i40iw_status_code i40iw_sc_static_hmc_pages_allocated(
3817                                         struct i40iw_sc_cqp *cqp,
3818                                         u64 scratch,
3819                                         u8 hmc_fn_id,
3820                                         bool post_sq,
3821                                         bool poll_registers)
3822 {
3823         u64 header;
3824         u64 *wqe;
3825         u32 tail, val, error;
3826         enum i40iw_status_code ret_code = 0;
3827
3828         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3829         if (!wqe)
3830                 return I40IW_ERR_RING_FULL;
3831         set_64bit_val(wqe,
3832                       16,
3833                       LS_64(hmc_fn_id, I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID));
3834
3835         header = LS_64(I40IW_CQP_OP_SHMC_PAGES_ALLOCATED, I40IW_CQPSQ_OPCODE) |
3836                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3837
3838         i40iw_insert_wqe_hdr(wqe, header);
3839
3840         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SHMC_PAGES_ALLOCATED WQE",
3841                         wqe, I40IW_CQP_WQE_SIZE * 8);
3842         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
3843         if (error) {
3844                 ret_code = I40IW_ERR_CQP_COMPL_ERROR;
3845                 return ret_code;
3846         }
3847         if (post_sq) {
3848                 i40iw_sc_cqp_post_sq(cqp);
3849                 if (poll_registers)
3850                         /* check for cqp sq tail update */
3851                         ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);
3852                 else
3853                         ret_code = i40iw_sc_poll_for_cqp_op_done(cqp,
3854                                                                  I40IW_CQP_OP_SHMC_PAGES_ALLOCATED,
3855                                                                  NULL);
3856         }
3857
3858         return ret_code;
3859 }
3860
3861 /**
3862  * i40iw_ring_full - check if cqp ring is full
3863  * @cqp: struct for cqp hw
3864  */
3865 static bool i40iw_ring_full(struct i40iw_sc_cqp *cqp)
3866 {
3867         return I40IW_RING_FULL_ERR(cqp->sq_ring);
3868 }
3869
3870 /**
3871  * i40iw_est_sd - returns approximate number of SDs for HMC
3872  * @dev: sc device struct
3873  * @hmc_info: hmc structure, size and count for HMC objects
3874  */
3875 static u64 i40iw_est_sd(struct i40iw_sc_dev *dev, struct i40iw_hmc_info *hmc_info)
3876 {
3877         int i;
3878         u64 size = 0;
3879         u64 sd;
3880
3881         for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_PBLE; i++)
3882                 size += hmc_info->hmc_obj[i].cnt * hmc_info->hmc_obj[i].size;
3883
3884         if (dev->is_pf)
3885                 size += hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size;
3886
3887         if (size & 0x1FFFFF)
3888                 sd = (size >> 21) + 1; /* add 1 for remainder */
3889         else
3890                 sd = size >> 21;
3891
3892         if (!dev->is_pf) {
3893                 /* 2MB alignment for VF PBLE HMC */
3894                 size = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size;
3895                 if (size & 0x1FFFFF)
3896                         sd += (size >> 21) + 1; /* add 1 for remainder */
3897                 else
3898                         sd += size >> 21;
3899         }
3900
3901         return sd;
3902 }
3903
3904 /**
3905  * i40iw_config_fpm_values - configure HMC objects
3906  * @dev: sc device struct
3907  * @qp_count: desired qp count
3908  */
3909 enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_count)
3910 {
3911         struct i40iw_virt_mem virt_mem;
3912         u32 i, mem_size;
3913         u32 qpwantedoriginal, qpwanted, mrwanted, pblewanted;
3914         u64 sd_needed;
3915         u32 loop_count = 0;
3916
3917         struct i40iw_hmc_info *hmc_info;
3918         struct i40iw_hmc_fpm_misc *hmc_fpm_misc;
3919         enum i40iw_status_code ret_code = 0;
3920
3921         hmc_info = dev->hmc_info;
3922         hmc_fpm_misc = &dev->hmc_fpm_misc;
3923
3924         ret_code = i40iw_sc_init_iw_hmc(dev, dev->hmc_fn_id);
3925         if (ret_code) {
3926                 i40iw_debug(dev, I40IW_DEBUG_HMC,
3927                             "i40iw_sc_init_iw_hmc returned error_code = %d\n",
3928                             ret_code);
3929                 return ret_code;
3930         }
3931
3932         for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_MAX; i++)
3933                 hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt;
3934         sd_needed = i40iw_est_sd(dev, hmc_info);
3935         i40iw_debug(dev, I40IW_DEBUG_HMC,
3936                     "%s: FW initial max sd_count[%08lld] first_sd_index[%04d]\n",
3937                     __func__, sd_needed, hmc_info->first_sd_index);
3938         i40iw_debug(dev, I40IW_DEBUG_HMC,
3939                     "%s: sd count %d where max sd is %d\n",
3940                     __func__, hmc_info->sd_table.sd_cnt,
3941                     hmc_fpm_misc->max_sds);
3942
3943         qpwanted = min(qp_count, hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt);
3944         qpwantedoriginal = qpwanted;
3945         mrwanted = hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt;
3946         pblewanted = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt;
3947
3948         i40iw_debug(dev, I40IW_DEBUG_HMC,
3949                     "req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d\n",
3950                     qp_count, hmc_fpm_misc->max_sds,
3951                     hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt,
3952                     hmc_info->hmc_obj[I40IW_HMC_IW_CQ].max_cnt,
3953                     hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt,
3954                     hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt);
3955
3956         do {
3957                 ++loop_count;
3958                 hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt = qpwanted;
3959                 hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt =
3960                         min(2 * qpwanted, hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt);
3961                 hmc_info->hmc_obj[I40IW_HMC_IW_SRQ].cnt = 0x00; /* Reserved */
3962                 hmc_info->hmc_obj[I40IW_HMC_IW_HTE].cnt =
3963                                         qpwanted * hmc_fpm_misc->ht_multiplier;
3964                 hmc_info->hmc_obj[I40IW_HMC_IW_ARP].cnt =
3965                         hmc_info->hmc_obj[I40IW_HMC_IW_ARP].max_cnt;
3966                 hmc_info->hmc_obj[I40IW_HMC_IW_APBVT_ENTRY].cnt = 1;
3967                 hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt = mrwanted;
3968
3969                 hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt =
3970                         roundup_pow_of_two(I40IW_MAX_WQ_ENTRIES * qpwanted);
3971                 hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt =
3972                         roundup_pow_of_two(2 * I40IW_MAX_IRD_SIZE * qpwanted);
3973                 hmc_info->hmc_obj[I40IW_HMC_IW_XFFL].cnt =
3974                         hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size;
3975                 hmc_info->hmc_obj[I40IW_HMC_IW_Q1FL].cnt =
3976                         hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size;
3977                 hmc_info->hmc_obj[I40IW_HMC_IW_TIMER].cnt =
3978                         ((qpwanted) / 512 + 1) * hmc_fpm_misc->timer_bucket;
3979                 hmc_info->hmc_obj[I40IW_HMC_IW_FSIMC].cnt = 0x00;
3980                 hmc_info->hmc_obj[I40IW_HMC_IW_FSIAV].cnt = 0x00;
3981                 hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt = pblewanted;
3982
3983                 /* How much memory is needed for all the objects. */
3984                 sd_needed = i40iw_est_sd(dev, hmc_info);
3985                 if ((loop_count > 1000) ||
3986                     ((!(loop_count % 10)) &&
3987                     (qpwanted > qpwantedoriginal * 2 / 3))) {
3988                         if (qpwanted > FPM_MULTIPLIER)
3989                                 qpwanted = roundup_pow_of_two(qpwanted -
3990                                                               FPM_MULTIPLIER);
3991                         qpwanted >>= 1;
3992                 }
3993                 if (mrwanted > FPM_MULTIPLIER * 10)
3994                         mrwanted -= FPM_MULTIPLIER * 10;
3995                 if (pblewanted > FPM_MULTIPLIER * 1000)
3996                         pblewanted -= FPM_MULTIPLIER * 1000;
3997         } while (sd_needed > hmc_fpm_misc->max_sds && loop_count < 2000);
3998
3999         i40iw_debug(dev, I40IW_DEBUG_HMC,
4000                     "loop_cnt=%d, sd_needed=%lld, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d\n",
4001                     loop_count, sd_needed,
4002                     hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt,
4003                     hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt,
4004                     hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt,
4005                     hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt);
4006
4007         ret_code = i40iw_sc_configure_iw_fpm(dev, dev->hmc_fn_id);
4008         if (ret_code) {
4009                 i40iw_debug(dev, I40IW_DEBUG_HMC,
4010                             "configure_iw_fpm returned error_code[x%08X]\n",
4011                             i40iw_rd32(dev->hw, dev->is_pf ? I40E_PFPE_CQPERRCODES : I40E_VFPE_CQPERRCODES1));
4012                 return ret_code;
4013         }
4014
4015         mem_size = sizeof(struct i40iw_hmc_sd_entry) *
4016                    (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1);
4017         ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size);
4018         if (ret_code) {
4019                 i40iw_debug(dev, I40IW_DEBUG_HMC,
4020                             "%s: failed to allocate memory for sd_entry buffer\n",
4021                             __func__);
4022                 return ret_code;
4023         }
4024         hmc_info->sd_table.sd_entry = virt_mem.va;
4025
4026         return ret_code;
4027 }
4028
4029 /**
4030  * i40iw_exec_cqp_cmd - execute cqp cmd when wqe are available
4031  * @dev: rdma device
4032  * @pcmdinfo: cqp command info
4033  */
4034 static enum i40iw_status_code i40iw_exec_cqp_cmd(struct i40iw_sc_dev *dev,
4035                                                  struct cqp_commands_info *pcmdinfo)
4036 {
4037         enum i40iw_status_code status;
4038         struct i40iw_dma_mem values_mem;
4039
4040         dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++;
4041         switch (pcmdinfo->cqp_cmd) {
4042         case OP_DELETE_LOCAL_MAC_IPADDR_ENTRY:
4043                 status = i40iw_sc_del_local_mac_ipaddr_entry(
4044                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.cqp,
4045                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.scratch,
4046                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.entry_idx,
4047                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.ignore_ref_count,
4048                                 pcmdinfo->post_sq);
4049                 break;
4050         case OP_CEQ_DESTROY:
4051                 status = i40iw_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,
4052                                               pcmdinfo->in.u.ceq_destroy.scratch,
4053                                               pcmdinfo->post_sq);
4054                 break;
4055         case OP_AEQ_DESTROY:
4056                 status = i40iw_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq,
4057                                               pcmdinfo->in.u.aeq_destroy.scratch,
4058                                               pcmdinfo->post_sq);
4059
4060                 break;
4061         case OP_DELETE_ARP_CACHE_ENTRY:
4062                 status = i40iw_sc_del_arp_cache_entry(
4063                                 pcmdinfo->in.u.del_arp_cache_entry.cqp,
4064                                 pcmdinfo->in.u.del_arp_cache_entry.scratch,
4065                                 pcmdinfo->in.u.del_arp_cache_entry.arp_index,
4066                                 pcmdinfo->post_sq);
4067                 break;
4068         case OP_MANAGE_APBVT_ENTRY:
4069                 status = i40iw_sc_manage_apbvt_entry(
4070                                 pcmdinfo->in.u.manage_apbvt_entry.cqp,
4071                                 &pcmdinfo->in.u.manage_apbvt_entry.info,
4072                                 pcmdinfo->in.u.manage_apbvt_entry.scratch,
4073                                 pcmdinfo->post_sq);
4074                 break;
4075         case OP_CEQ_CREATE:
4076                 status = i40iw_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,
4077                                              pcmdinfo->in.u.ceq_create.scratch,
4078                                              pcmdinfo->post_sq);
4079                 break;
4080         case OP_AEQ_CREATE:
4081                 status = i40iw_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq,
4082                                              pcmdinfo->in.u.aeq_create.scratch,
4083                                              pcmdinfo->post_sq);
4084                 break;
4085         case OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY:
4086                 status = i40iw_sc_alloc_local_mac_ipaddr_entry(
4087                                 pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.cqp,
4088                                 pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.scratch,
4089                                 pcmdinfo->post_sq);
4090                 break;
4091         case OP_ADD_LOCAL_MAC_IPADDR_ENTRY:
4092                 status = i40iw_sc_add_local_mac_ipaddr_entry(
4093                                 pcmdinfo->in.u.add_local_mac_ipaddr_entry.cqp,
4094                                 &pcmdinfo->in.u.add_local_mac_ipaddr_entry.info,
4095                                 pcmdinfo->in.u.add_local_mac_ipaddr_entry.scratch,
4096                                 pcmdinfo->post_sq);
4097                 break;
4098         case OP_MANAGE_QHASH_TABLE_ENTRY:
4099                 status = i40iw_sc_manage_qhash_table_entry(
4100                                 pcmdinfo->in.u.manage_qhash_table_entry.cqp,
4101                                 &pcmdinfo->in.u.manage_qhash_table_entry.info,
4102                                 pcmdinfo->in.u.manage_qhash_table_entry.scratch,
4103                                 pcmdinfo->post_sq);
4104
4105                 break;
4106         case OP_QP_MODIFY:
4107                 status = i40iw_sc_qp_modify(
4108                                 pcmdinfo->in.u.qp_modify.qp,
4109                                 &pcmdinfo->in.u.qp_modify.info,
4110                                 pcmdinfo->in.u.qp_modify.scratch,
4111                                 pcmdinfo->post_sq);
4112
4113                 break;
4114         case OP_QP_UPLOAD_CONTEXT:
4115                 status = i40iw_sc_qp_upload_context(
4116                                 pcmdinfo->in.u.qp_upload_context.dev,
4117                                 &pcmdinfo->in.u.qp_upload_context.info,
4118                                 pcmdinfo->in.u.qp_upload_context.scratch,
4119                                 pcmdinfo->post_sq);
4120
4121                 break;
4122         case OP_CQ_CREATE:
4123                 status = i40iw_sc_cq_create(
4124                                 pcmdinfo->in.u.cq_create.cq,
4125                                 pcmdinfo->in.u.cq_create.scratch,
4126                                 pcmdinfo->in.u.cq_create.check_overflow,
4127                                 pcmdinfo->post_sq);
4128                 break;
4129         case OP_CQ_DESTROY:
4130                 status = i40iw_sc_cq_destroy(
4131                                 pcmdinfo->in.u.cq_destroy.cq,
4132                                 pcmdinfo->in.u.cq_destroy.scratch,
4133                                 pcmdinfo->post_sq);
4134
4135                 break;
4136         case OP_QP_CREATE:
4137                 status = i40iw_sc_qp_create(
4138                                 pcmdinfo->in.u.qp_create.qp,
4139                                 &pcmdinfo->in.u.qp_create.info,
4140                                 pcmdinfo->in.u.qp_create.scratch,
4141                                 pcmdinfo->post_sq);
4142                 break;
4143         case OP_QP_DESTROY:
4144                 status = i40iw_sc_qp_destroy(
4145                                 pcmdinfo->in.u.qp_destroy.qp,
4146                                 pcmdinfo->in.u.qp_destroy.scratch,
4147                                 pcmdinfo->in.u.qp_destroy.remove_hash_idx,
4148                                 pcmdinfo->in.u.qp_destroy.
4149                                 ignore_mw_bnd,
4150                                 pcmdinfo->post_sq);
4151
4152                 break;
4153         case OP_ALLOC_STAG:
4154                 status = i40iw_sc_alloc_stag(
4155                                 pcmdinfo->in.u.alloc_stag.dev,
4156                                 &pcmdinfo->in.u.alloc_stag.info,
4157                                 pcmdinfo->in.u.alloc_stag.scratch,
4158                                 pcmdinfo->post_sq);
4159                 break;
4160         case OP_MR_REG_NON_SHARED:
4161                 status = i40iw_sc_mr_reg_non_shared(
4162                                 pcmdinfo->in.u.mr_reg_non_shared.dev,
4163                                 &pcmdinfo->in.u.mr_reg_non_shared.info,
4164                                 pcmdinfo->in.u.mr_reg_non_shared.scratch,
4165                                 pcmdinfo->post_sq);
4166
4167                 break;
4168         case OP_DEALLOC_STAG:
4169                 status = i40iw_sc_dealloc_stag(
4170                                 pcmdinfo->in.u.dealloc_stag.dev,
4171                                 &pcmdinfo->in.u.dealloc_stag.info,
4172                                 pcmdinfo->in.u.dealloc_stag.scratch,
4173                                 pcmdinfo->post_sq);
4174
4175                 break;
4176         case OP_MW_ALLOC:
4177                 status = i40iw_sc_mw_alloc(
4178                                 pcmdinfo->in.u.mw_alloc.dev,
4179                                 pcmdinfo->in.u.mw_alloc.scratch,
4180                                 pcmdinfo->in.u.mw_alloc.mw_stag_index,
4181                                 pcmdinfo->in.u.mw_alloc.pd_id,
4182                                 pcmdinfo->post_sq);
4183
4184                 break;
4185         case OP_QP_FLUSH_WQES:
4186                 status = i40iw_sc_qp_flush_wqes(
4187                                 pcmdinfo->in.u.qp_flush_wqes.qp,
4188                                 &pcmdinfo->in.u.qp_flush_wqes.info,
4189                                 pcmdinfo->in.u.qp_flush_wqes.
4190                                 scratch, pcmdinfo->post_sq);
4191                 break;
4192         case OP_GEN_AE:
4193                 status = i40iw_sc_gen_ae(
4194                                 pcmdinfo->in.u.gen_ae.qp,
4195                                 &pcmdinfo->in.u.gen_ae.info,
4196                                 pcmdinfo->in.u.gen_ae.scratch,
4197                                 pcmdinfo->post_sq);
4198                 break;
4199         case OP_ADD_ARP_CACHE_ENTRY:
4200                 status = i40iw_sc_add_arp_cache_entry(
4201                                 pcmdinfo->in.u.add_arp_cache_entry.cqp,
4202                                 &pcmdinfo->in.u.add_arp_cache_entry.info,
4203                                 pcmdinfo->in.u.add_arp_cache_entry.scratch,
4204                                 pcmdinfo->post_sq);
4205                 break;
4206         case OP_MANAGE_PUSH_PAGE:
4207                 status = i40iw_sc_manage_push_page(
4208                                 pcmdinfo->in.u.manage_push_page.cqp,
4209                                 &pcmdinfo->in.u.manage_push_page.info,
4210                                 pcmdinfo->in.u.manage_push_page.scratch,
4211                                 pcmdinfo->post_sq);
4212                 break;
4213         case OP_UPDATE_PE_SDS:
4214                 /* case I40IW_CQP_OP_UPDATE_PE_SDS */
4215                 status = i40iw_update_pe_sds(
4216                                 pcmdinfo->in.u.update_pe_sds.dev,
4217                                 &pcmdinfo->in.u.update_pe_sds.info,
4218                                 pcmdinfo->in.u.update_pe_sds.
4219                                 scratch);
4220
4221                 break;
4222         case OP_MANAGE_HMC_PM_FUNC_TABLE:
4223                 status = i40iw_sc_manage_hmc_pm_func_table(
4224                                 pcmdinfo->in.u.manage_hmc_pm.dev->cqp,
4225                                 pcmdinfo->in.u.manage_hmc_pm.scratch,
4226                                 (u8)pcmdinfo->in.u.manage_hmc_pm.info.vf_id,
4227                                 pcmdinfo->in.u.manage_hmc_pm.info.free_fcn,
4228                                 true);
4229                 break;
4230         case OP_SUSPEND:
4231                 status = i40iw_sc_suspend_qp(
4232                                 pcmdinfo->in.u.suspend_resume.cqp,
4233                                 pcmdinfo->in.u.suspend_resume.qp,
4234                                 pcmdinfo->in.u.suspend_resume.scratch);
4235                 break;
4236         case OP_RESUME:
4237                 status = i40iw_sc_resume_qp(
4238                                 pcmdinfo->in.u.suspend_resume.cqp,
4239                                 pcmdinfo->in.u.suspend_resume.qp,
4240                                 pcmdinfo->in.u.suspend_resume.scratch);
4241                 break;
4242         case OP_MANAGE_VF_PBLE_BP:
4243                 status = i40iw_manage_vf_pble_bp(
4244                                 pcmdinfo->in.u.manage_vf_pble_bp.cqp,
4245                                 &pcmdinfo->in.u.manage_vf_pble_bp.info,
4246                                 pcmdinfo->in.u.manage_vf_pble_bp.scratch, true);
4247                 break;
4248         case OP_QUERY_FPM_VALUES:
4249                 values_mem.pa = pcmdinfo->in.u.query_fpm_values.fpm_values_pa;
4250                 values_mem.va = pcmdinfo->in.u.query_fpm_values.fpm_values_va;
4251                 status = i40iw_sc_query_fpm_values(
4252                                 pcmdinfo->in.u.query_fpm_values.cqp,
4253                                 pcmdinfo->in.u.query_fpm_values.scratch,
4254                                 pcmdinfo->in.u.query_fpm_values.hmc_fn_id,
4255                                 &values_mem, true, I40IW_CQP_WAIT_EVENT);
4256                 break;
4257         case OP_COMMIT_FPM_VALUES:
4258                 values_mem.pa = pcmdinfo->in.u.commit_fpm_values.fpm_values_pa;
4259                 values_mem.va = pcmdinfo->in.u.commit_fpm_values.fpm_values_va;
4260                 status = i40iw_sc_commit_fpm_values(
4261                                 pcmdinfo->in.u.commit_fpm_values.cqp,
4262                                 pcmdinfo->in.u.commit_fpm_values.scratch,
4263                                 pcmdinfo->in.u.commit_fpm_values.hmc_fn_id,
4264                                 &values_mem,
4265                                 true,
4266                                 I40IW_CQP_WAIT_EVENT);
4267                 break;
4268         default:
4269                 status = I40IW_NOT_SUPPORTED;
4270                 break;
4271         }
4272
4273         return status;
4274 }
4275
4276 /**
4277  * i40iw_process_cqp_cmd - process all cqp commands
4278  * @dev: sc device struct
4279  * @pcmdinfo: cqp command info
4280  */
4281 enum i40iw_status_code i40iw_process_cqp_cmd(struct i40iw_sc_dev *dev,
4282                                              struct cqp_commands_info *pcmdinfo)
4283 {
4284         enum i40iw_status_code status = 0;
4285         unsigned long flags;
4286
4287         spin_lock_irqsave(&dev->cqp_lock, flags);
4288         if (list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp))
4289                 status = i40iw_exec_cqp_cmd(dev, pcmdinfo);
4290         else
4291                 list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head);
4292         spin_unlock_irqrestore(&dev->cqp_lock, flags);
4293         return status;
4294 }
4295
4296 /**
4297  * i40iw_process_bh - called from tasklet for cqp list
4298  * @dev: sc device struct
4299  */
4300 enum i40iw_status_code i40iw_process_bh(struct i40iw_sc_dev *dev)
4301 {
4302         enum i40iw_status_code status = 0;
4303         struct cqp_commands_info *pcmdinfo;
4304         unsigned long flags;
4305
4306         spin_lock_irqsave(&dev->cqp_lock, flags);
4307         while (!list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp)) {
4308                 pcmdinfo = (struct cqp_commands_info *)i40iw_remove_head(&dev->cqp_cmd_head);
4309
4310                 status = i40iw_exec_cqp_cmd(dev, pcmdinfo);
4311                 if (status)
4312                         break;
4313         }
4314         spin_unlock_irqrestore(&dev->cqp_lock, flags);
4315         return status;
4316 }
4317
4318 /**
4319  * i40iw_iwarp_opcode - determine if incoming is rdma layer
4320  * @info: aeq info for the packet
4321  * @pkt: packet for error
4322  */
4323 static u32 i40iw_iwarp_opcode(struct i40iw_aeqe_info *info, u8 *pkt)
4324 {
4325         __be16 *mpa;
4326         u32 opcode = 0xffffffff;
4327
4328         if (info->q2_data_written) {
4329                 mpa = (__be16 *)pkt;
4330                 opcode = ntohs(mpa[1]) & 0xf;
4331         }
4332         return opcode;
4333 }
4334
4335 /**
4336  * i40iw_locate_mpa - return pointer to mpa in the pkt
4337  * @pkt: packet with data
4338  */
4339 static u8 *i40iw_locate_mpa(u8 *pkt)
4340 {
4341         /* skip over ethernet header */
4342         pkt += I40IW_MAC_HLEN;
4343
4344         /* Skip over IP and TCP headers */
4345         pkt += 4 * (pkt[0] & 0x0f);
4346         pkt += 4 * ((pkt[12] >> 4) & 0x0f);
4347         return pkt;
4348 }
4349
4350 /**
4351  * i40iw_setup_termhdr - termhdr for terminate pkt
4352  * @qp: sc qp ptr for pkt
4353  * @hdr: term hdr
4354  * @opcode: flush opcode for termhdr
4355  * @layer_etype: error layer + error type
4356  * @err: error cod ein the header
4357  */
4358 static void i40iw_setup_termhdr(struct i40iw_sc_qp *qp,
4359                                 struct i40iw_terminate_hdr *hdr,
4360                                 enum i40iw_flush_opcode opcode,
4361                                 u8 layer_etype,
4362                                 u8 err)
4363 {
4364         qp->flush_code = opcode;
4365         hdr->layer_etype = layer_etype;
4366         hdr->error_code = err;
4367 }
4368
4369 /**
4370  * i40iw_bld_terminate_hdr - build terminate message header
4371  * @qp: qp associated with received terminate AE
4372  * @info: the struct contiaing AE information
4373  */
4374 static int i40iw_bld_terminate_hdr(struct i40iw_sc_qp *qp,
4375                                    struct i40iw_aeqe_info *info)
4376 {
4377         u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
4378         u16 ddp_seg_len;
4379         int copy_len = 0;
4380         u8 is_tagged = 0;
4381         u32 opcode;
4382         struct i40iw_terminate_hdr *termhdr;
4383
4384         termhdr = (struct i40iw_terminate_hdr *)qp->q2_buf;
4385         memset(termhdr, 0, Q2_BAD_FRAME_OFFSET);
4386
4387         if (info->q2_data_written) {
4388                 /* Use data from offending packet to fill in ddp & rdma hdrs */
4389                 pkt = i40iw_locate_mpa(pkt);
4390                 ddp_seg_len = ntohs(*(__be16 *)pkt);
4391                 if (ddp_seg_len) {
4392                         copy_len = 2;
4393                         termhdr->hdrct = DDP_LEN_FLAG;
4394                         if (pkt[2] & 0x80) {
4395                                 is_tagged = 1;
4396                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
4397                                         copy_len += TERM_DDP_LEN_TAGGED;
4398                                         termhdr->hdrct |= DDP_HDR_FLAG;
4399                                 }
4400                         } else {
4401                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
4402                                         copy_len += TERM_DDP_LEN_UNTAGGED;
4403                                         termhdr->hdrct |= DDP_HDR_FLAG;
4404                                 }
4405
4406                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
4407                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
4408                                                 copy_len += TERM_RDMA_LEN;
4409                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
4410                                         }
4411                                 }
4412                         }
4413                 }
4414         }
4415
4416         opcode = i40iw_iwarp_opcode(info, pkt);
4417
4418         switch (info->ae_id) {
4419         case I40IW_AE_AMP_UNALLOCATED_STAG:
4420                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4421                 if (opcode == I40IW_OP_TYPE_RDMA_WRITE)
4422                         i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
4423                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_STAG);
4424                 else
4425                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4426                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG);
4427                 break;
4428         case I40IW_AE_AMP_BOUNDS_VIOLATION:
4429                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4430                 if (info->q2_data_written)
4431                         i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
4432                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_BOUNDS);
4433                 else
4434                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4435                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_BOUNDS);
4436                 break;
4437         case I40IW_AE_AMP_BAD_PD:
4438                 switch (opcode) {
4439                 case I40IW_OP_TYPE_RDMA_WRITE:
4440                         i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
4441                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_UNASSOC_STAG);
4442                         break;
4443                 case I40IW_OP_TYPE_SEND_INV:
4444                 case I40IW_OP_TYPE_SEND_SOL_INV:
4445                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4446                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_CANT_INV_STAG);
4447                         break;
4448                 default:
4449                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4450                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_UNASSOC_STAG);
4451                 }
4452                 break;
4453         case I40IW_AE_AMP_INVALID_STAG:
4454                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4455                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4456                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG);
4457                 break;
4458         case I40IW_AE_AMP_BAD_QP:
4459                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
4460                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);
4461                 break;
4462         case I40IW_AE_AMP_BAD_STAG_KEY:
4463         case I40IW_AE_AMP_BAD_STAG_INDEX:
4464                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4465                 switch (opcode) {
4466                 case I40IW_OP_TYPE_SEND_INV:
4467                 case I40IW_OP_TYPE_SEND_SOL_INV:
4468                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,
4469                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_CANT_INV_STAG);
4470                         break;
4471                 default:
4472                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4473                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_INV_STAG);
4474                 }
4475                 break;
4476         case I40IW_AE_AMP_RIGHTS_VIOLATION:
4477         case I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
4478         case I40IW_AE_PRIV_OPERATION_DENIED:
4479                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4480                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4481                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_ACCESS);
4482                 break;
4483         case I40IW_AE_AMP_TO_WRAP:
4484                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4485                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4486                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_TO_WRAP);
4487                 break;
4488         case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:
4489                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4490                                     (LAYER_MPA << 4) | DDP_LLP, MPA_CRC);
4491                 break;
4492         case I40IW_AE_LLP_SEGMENT_TOO_LARGE:
4493         case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
4494                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,
4495                                     (LAYER_DDP << 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);
4496                 break;
4497         case I40IW_AE_LCE_QP_CATASTROPHIC:
4498         case I40IW_AE_DDP_NO_L_BIT:
4499                 i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,
4500                                     (LAYER_DDP << 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);
4501                 break;
4502         case I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN:
4503                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4504                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_RANGE);
4505                 break;
4506         case I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
4507                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4508                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,
4509                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_TOO_LONG);
4510                 break;
4511         case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:
4512                 if (is_tagged)
4513                         i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4514                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_DDP_VER);
4515                 else
4516                         i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4517                                             (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_DDP_VER);
4518                 break;
4519         case I40IW_AE_DDP_UBE_INVALID_MO:
4520                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4521                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MO);
4522                 break;
4523         case I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
4524                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,
4525                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_NO_BUF);
4526                 break;
4527         case I40IW_AE_DDP_UBE_INVALID_QN:
4528                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4529                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);
4530                 break;
4531         case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
4532                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4533                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_INV_RDMAP_VER);
4534                 break;
4535         case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
4536                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
4537                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_UNEXPECTED_OP);
4538                 break;
4539         default:
4540                 i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,
4541                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_UNSPECIFIED);
4542                 break;
4543         }
4544
4545         if (copy_len)
4546                 memcpy(termhdr + 1, pkt, copy_len);
4547
4548         return sizeof(struct i40iw_terminate_hdr) + copy_len;
4549 }
4550
4551 /**
4552  * i40iw_terminate_send_fin() - Send fin for terminate message
4553  * @qp: qp associated with received terminate AE
4554  */
4555 void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp)
4556 {
4557         /* Send the fin only */
4558         i40iw_term_modify_qp(qp,
4559                              I40IW_QP_STATE_TERMINATE,
4560                              I40IWQP_TERM_SEND_FIN_ONLY,
4561                              0);
4562 }
4563
4564 /**
4565  * i40iw_terminate_connection() - Bad AE and send terminate to remote QP
4566  * @qp: qp associated with received terminate AE
4567  * @info: the struct contiaing AE information
4568  */
4569 void i40iw_terminate_connection(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info)
4570 {
4571         u8 termlen = 0;
4572
4573         if (qp->term_flags & I40IW_TERM_SENT)
4574                 return;         /* Sanity check */
4575
4576         /* Eventtype can change from bld_terminate_hdr */
4577         qp->eventtype = TERM_EVENT_QP_FATAL;
4578         termlen = i40iw_bld_terminate_hdr(qp, info);
4579         i40iw_terminate_start_timer(qp);
4580         qp->term_flags |= I40IW_TERM_SENT;
4581         i40iw_term_modify_qp(qp, I40IW_QP_STATE_TERMINATE,
4582                              I40IWQP_TERM_SEND_TERM_ONLY, termlen);
4583 }
4584
4585 /**
4586  * i40iw_terminate_received - handle terminate received AE
4587  * @qp: qp associated with received terminate AE
4588  * @info: the struct contiaing AE information
4589  */
4590 void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info)
4591 {
4592         u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
4593         __be32 *mpa;
4594         u8 ddp_ctl;
4595         u8 rdma_ctl;
4596         u16 aeq_id = 0;
4597         struct i40iw_terminate_hdr *termhdr;
4598
4599         mpa = (__be32 *)i40iw_locate_mpa(pkt);
4600         if (info->q2_data_written) {
4601                 /* did not validate the frame - do it now */
4602                 ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff;
4603                 rdma_ctl = ntohl(mpa[0]) & 0xff;
4604                 if ((ddp_ctl & 0xc0) != 0x40)
4605                         aeq_id = I40IW_AE_LCE_QP_CATASTROPHIC;
4606                 else if ((ddp_ctl & 0x03) != 1)
4607                         aeq_id = I40IW_AE_DDP_UBE_INVALID_DDP_VERSION;
4608                 else if (ntohl(mpa[2]) != 2)
4609                         aeq_id = I40IW_AE_DDP_UBE_INVALID_QN;
4610                 else if (ntohl(mpa[3]) != 1)
4611                         aeq_id = I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN;
4612                 else if (ntohl(mpa[4]) != 0)
4613                         aeq_id = I40IW_AE_DDP_UBE_INVALID_MO;
4614                 else if ((rdma_ctl & 0xc0) != 0x40)
4615                         aeq_id = I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;
4616
4617                 info->ae_id = aeq_id;
4618                 if (info->ae_id) {
4619                         /* Bad terminate recvd - send back a terminate */
4620                         i40iw_terminate_connection(qp, info);
4621                         return;
4622                 }
4623         }
4624
4625         qp->term_flags |= I40IW_TERM_RCVD;
4626         qp->eventtype = TERM_EVENT_QP_FATAL;
4627         termhdr = (struct i40iw_terminate_hdr *)&mpa[5];
4628         if (termhdr->layer_etype == RDMAP_REMOTE_PROT ||
4629             termhdr->layer_etype == RDMAP_REMOTE_OP) {
4630                 i40iw_terminate_done(qp, 0);
4631         } else {
4632                 i40iw_terminate_start_timer(qp);
4633                 i40iw_terminate_send_fin(qp);
4634         }
4635 }
4636
4637 /**
4638  * i40iw_sc_vsi_init - Initialize virtual device
4639  * @vsi: pointer to the vsi structure
4640  * @info: parameters to initialize vsi
4641  **/
4642 void i40iw_sc_vsi_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_init_info *info)
4643 {
4644         int i;
4645
4646         vsi->dev = info->dev;
4647         vsi->back_vsi = info->back_vsi;
4648         vsi->mtu = info->params->mtu;
4649         vsi->exception_lan_queue = info->exception_lan_queue;
4650         i40iw_fill_qos_list(info->params->qs_handle_list);
4651
4652         for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
4653                 vsi->qos[i].qs_handle = info->params->qs_handle_list[i];
4654                 i40iw_debug(vsi->dev, I40IW_DEBUG_DCB, "qset[%d]: %d\n", i,
4655                             vsi->qos[i].qs_handle);
4656                 spin_lock_init(&vsi->qos[i].lock);
4657                 INIT_LIST_HEAD(&vsi->qos[i].qplist);
4658         }
4659 }
4660
4661 /**
4662  * i40iw_hw_stats_init - Initiliaze HW stats table
4663  * @stats: pestat struct
4664  * @fcn_idx: PCI fn id
4665  * @is_pf: Is it a PF?
4666  *
4667  * Populate the HW stats table with register offset addr for each
4668  * stats. And start the perioidic stats timer.
4669  */
4670 void i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 fcn_idx, bool is_pf)
4671 {
4672         u32 stats_reg_offset;
4673         u32 stats_index;
4674         struct i40iw_dev_hw_stats_offsets *stats_table =
4675                 &stats->hw_stats_offsets;
4676         struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
4677
4678         if (is_pf) {
4679                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
4680                                 I40E_GLPES_PFIP4RXDISCARD(fcn_idx);
4681                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
4682                                 I40E_GLPES_PFIP4RXTRUNC(fcn_idx);
4683                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
4684                                 I40E_GLPES_PFIP4TXNOROUTE(fcn_idx);
4685                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
4686                                 I40E_GLPES_PFIP6RXDISCARD(fcn_idx);
4687                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
4688                                 I40E_GLPES_PFIP6RXTRUNC(fcn_idx);
4689                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
4690                                 I40E_GLPES_PFIP6TXNOROUTE(fcn_idx);
4691                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
4692                                 I40E_GLPES_PFTCPRTXSEG(fcn_idx);
4693                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
4694                                 I40E_GLPES_PFTCPRXOPTERR(fcn_idx);
4695                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
4696                                 I40E_GLPES_PFTCPRXPROTOERR(fcn_idx);
4697
4698                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
4699                                 I40E_GLPES_PFIP4RXOCTSLO(fcn_idx);
4700                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
4701                                 I40E_GLPES_PFIP4RXPKTSLO(fcn_idx);
4702                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
4703                                 I40E_GLPES_PFIP4RXFRAGSLO(fcn_idx);
4704                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
4705                                 I40E_GLPES_PFIP4RXMCPKTSLO(fcn_idx);
4706                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
4707                                 I40E_GLPES_PFIP4TXOCTSLO(fcn_idx);
4708                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
4709                                 I40E_GLPES_PFIP4TXPKTSLO(fcn_idx);
4710                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
4711                                 I40E_GLPES_PFIP4TXFRAGSLO(fcn_idx);
4712                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
4713                                 I40E_GLPES_PFIP4TXMCPKTSLO(fcn_idx);
4714                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
4715                                 I40E_GLPES_PFIP6RXOCTSLO(fcn_idx);
4716                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
4717                                 I40E_GLPES_PFIP6RXPKTSLO(fcn_idx);
4718                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
4719                                 I40E_GLPES_PFIP6RXFRAGSLO(fcn_idx);
4720                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
4721                                 I40E_GLPES_PFIP6RXMCPKTSLO(fcn_idx);
4722                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
4723                                 I40E_GLPES_PFIP6TXOCTSLO(fcn_idx);
4724                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4725                                 I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
4726                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4727                                 I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
4728                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
4729                                 I40E_GLPES_PFIP6TXFRAGSLO(fcn_idx);
4730                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
4731                                 I40E_GLPES_PFTCPRXSEGSLO(fcn_idx);
4732                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
4733                                 I40E_GLPES_PFTCPTXSEGLO(fcn_idx);
4734                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
4735                                 I40E_GLPES_PFRDMARXRDSLO(fcn_idx);
4736                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
4737                                 I40E_GLPES_PFRDMARXSNDSLO(fcn_idx);
4738                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
4739                                 I40E_GLPES_PFRDMARXWRSLO(fcn_idx);
4740                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
4741                                 I40E_GLPES_PFRDMATXRDSLO(fcn_idx);
4742                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
4743                                 I40E_GLPES_PFRDMATXSNDSLO(fcn_idx);
4744                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
4745                                 I40E_GLPES_PFRDMATXWRSLO(fcn_idx);
4746                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
4747                                 I40E_GLPES_PFRDMAVBNDLO(fcn_idx);
4748                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
4749                                 I40E_GLPES_PFRDMAVINVLO(fcn_idx);
4750         } else {
4751                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
4752                                 I40E_GLPES_VFIP4RXDISCARD(fcn_idx);
4753                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
4754                                 I40E_GLPES_VFIP4RXTRUNC(fcn_idx);
4755                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
4756                                 I40E_GLPES_VFIP4TXNOROUTE(fcn_idx);
4757                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
4758                                 I40E_GLPES_VFIP6RXDISCARD(fcn_idx);
4759                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
4760                                 I40E_GLPES_VFIP6RXTRUNC(fcn_idx);
4761                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
4762                                 I40E_GLPES_VFIP6TXNOROUTE(fcn_idx);
4763                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
4764                                 I40E_GLPES_VFTCPRTXSEG(fcn_idx);
4765                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
4766                                 I40E_GLPES_VFTCPRXOPTERR(fcn_idx);
4767                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
4768                                 I40E_GLPES_VFTCPRXPROTOERR(fcn_idx);
4769
4770                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
4771                                 I40E_GLPES_VFIP4RXOCTSLO(fcn_idx);
4772                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
4773                                 I40E_GLPES_VFIP4RXPKTSLO(fcn_idx);
4774                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
4775                                 I40E_GLPES_VFIP4RXFRAGSLO(fcn_idx);
4776                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
4777                                 I40E_GLPES_VFIP4RXMCPKTSLO(fcn_idx);
4778                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
4779                                 I40E_GLPES_VFIP4TXOCTSLO(fcn_idx);
4780                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
4781                                 I40E_GLPES_VFIP4TXPKTSLO(fcn_idx);
4782                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
4783                                 I40E_GLPES_VFIP4TXFRAGSLO(fcn_idx);
4784                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
4785                                 I40E_GLPES_VFIP4TXMCPKTSLO(fcn_idx);
4786                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
4787                                 I40E_GLPES_VFIP6RXOCTSLO(fcn_idx);
4788                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
4789                                 I40E_GLPES_VFIP6RXPKTSLO(fcn_idx);
4790                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
4791                                 I40E_GLPES_VFIP6RXFRAGSLO(fcn_idx);
4792                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
4793                                 I40E_GLPES_VFIP6RXMCPKTSLO(fcn_idx);
4794                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
4795                                 I40E_GLPES_VFIP6TXOCTSLO(fcn_idx);
4796                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4797                                 I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
4798                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4799                                 I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
4800                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
4801                                 I40E_GLPES_VFIP6TXFRAGSLO(fcn_idx);
4802                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
4803                                 I40E_GLPES_VFTCPRXSEGSLO(fcn_idx);
4804                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
4805                                 I40E_GLPES_VFTCPTXSEGLO(fcn_idx);
4806                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
4807                                 I40E_GLPES_VFRDMARXRDSLO(fcn_idx);
4808                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
4809                                 I40E_GLPES_VFRDMARXSNDSLO(fcn_idx);
4810                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
4811                                 I40E_GLPES_VFRDMARXWRSLO(fcn_idx);
4812                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
4813                                 I40E_GLPES_VFRDMATXRDSLO(fcn_idx);
4814                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
4815                                 I40E_GLPES_VFRDMATXSNDSLO(fcn_idx);
4816                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
4817                                 I40E_GLPES_VFRDMATXWRSLO(fcn_idx);
4818                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
4819                                 I40E_GLPES_VFRDMAVBNDLO(fcn_idx);
4820                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
4821                                 I40E_GLPES_VFRDMAVINVLO(fcn_idx);
4822         }
4823
4824         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
4825              stats_index++) {
4826                 stats_reg_offset = stats_table->stats_offset_64[stats_index];
4827                 last_rd_stats->stats_value_64[stats_index] =
4828                         readq(stats->hw->hw_addr + stats_reg_offset);
4829         }
4830
4831         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
4832              stats_index++) {
4833                 stats_reg_offset = stats_table->stats_offset_32[stats_index];
4834                 last_rd_stats->stats_value_32[stats_index] =
4835                         i40iw_rd32(stats->hw, stats_reg_offset);
4836         }
4837 }
4838
4839 /**
4840  * i40iw_hw_stats_read_32 - Read 32-bit HW stats counters and accommodates for roll-overs.
4841  * @stat: pestat struct
4842  * @index: index in HW stats table which contains offset reg-addr
4843  * @value: hw stats value
4844  */
4845 void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats,
4846                             enum i40iw_hw_stats_index_32b index,
4847                             u64 *value)
4848 {
4849         struct i40iw_dev_hw_stats_offsets *stats_table =
4850                 &stats->hw_stats_offsets;
4851         struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
4852         struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
4853         u64 new_stats_value = 0;
4854         u32 stats_reg_offset = stats_table->stats_offset_32[index];
4855
4856         new_stats_value = i40iw_rd32(stats->hw, stats_reg_offset);
4857         /*roll-over case */
4858         if (new_stats_value < last_rd_stats->stats_value_32[index])
4859                 hw_stats->stats_value_32[index] += new_stats_value;
4860         else
4861                 hw_stats->stats_value_32[index] +=
4862                         new_stats_value - last_rd_stats->stats_value_32[index];
4863         last_rd_stats->stats_value_32[index] = new_stats_value;
4864         *value = hw_stats->stats_value_32[index];
4865 }
4866
4867 /**
4868  * i40iw_hw_stats_read_64 - Read HW stats counters (greater than 32-bit) and accommodates for roll-overs.
4869  * @stats: pestat struct
4870  * @index: index in HW stats table which contains offset reg-addr
4871  * @value: hw stats value
4872  */
4873 void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats,
4874                             enum i40iw_hw_stats_index_64b index,
4875                             u64 *value)
4876 {
4877         struct i40iw_dev_hw_stats_offsets *stats_table =
4878                 &stats->hw_stats_offsets;
4879         struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
4880         struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
4881         u64 new_stats_value = 0;
4882         u32 stats_reg_offset = stats_table->stats_offset_64[index];
4883
4884         new_stats_value = readq(stats->hw->hw_addr + stats_reg_offset);
4885         /*roll-over case */
4886         if (new_stats_value < last_rd_stats->stats_value_64[index])
4887                 hw_stats->stats_value_64[index] += new_stats_value;
4888         else
4889                 hw_stats->stats_value_64[index] +=
4890                         new_stats_value - last_rd_stats->stats_value_64[index];
4891         last_rd_stats->stats_value_64[index] = new_stats_value;
4892         *value = hw_stats->stats_value_64[index];
4893 }
4894
4895 /**
4896  * i40iw_hw_stats_read_all - read all HW stat counters
4897  * @stats: pestat struct
4898  * @stats_values: hw stats structure
4899  *
4900  * Read all the HW stat counters and populates hw_stats structure
4901  * of passed-in vsi's pestat as well as copy created in stat_values.
4902  */
4903 void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat *stats,
4904                              struct i40iw_dev_hw_stats *stats_values)
4905 {
4906         u32 stats_index;
4907         unsigned long flags;
4908
4909         spin_lock_irqsave(&stats->lock, flags);
4910
4911         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
4912              stats_index++)
4913                 i40iw_hw_stats_read_32(stats, stats_index,
4914                                        &stats_values->stats_value_32[stats_index]);
4915         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
4916              stats_index++)
4917                 i40iw_hw_stats_read_64(stats, stats_index,
4918                                        &stats_values->stats_value_64[stats_index]);
4919         spin_unlock_irqrestore(&stats->lock, flags);
4920 }
4921
4922 /**
4923  * i40iw_hw_stats_refresh_all - Update all HW stats structs
4924  * @stats: pestat struct
4925  *
4926  * Read all the HW stats counters to refresh values in hw_stats structure
4927  * of passed-in dev's pestat
4928  */
4929 void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats)
4930 {
4931         u64 stats_value;
4932         u32 stats_index;
4933         unsigned long flags;
4934
4935         spin_lock_irqsave(&stats->lock, flags);
4936
4937         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
4938              stats_index++)
4939                 i40iw_hw_stats_read_32(stats, stats_index, &stats_value);
4940         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
4941              stats_index++)
4942                 i40iw_hw_stats_read_64(stats, stats_index, &stats_value);
4943         spin_unlock_irqrestore(&stats->lock, flags);
4944 }
4945
4946 /**
4947  * i40iw_get_fcn_id - Return the function id
4948  * @dev: pointer to the device
4949  */
4950 static u8 i40iw_get_fcn_id(struct i40iw_sc_dev *dev)
4951 {
4952         u8 fcn_id = I40IW_INVALID_FCN_ID;
4953         u8 i;
4954
4955         for (i = I40IW_FIRST_NON_PF_STAT; i < I40IW_MAX_STATS_COUNT; i++)
4956                 if (!dev->fcn_id_array[i]) {
4957                         fcn_id = i;
4958                         dev->fcn_id_array[i] = true;
4959                         break;
4960                 }
4961         return fcn_id;
4962 }
4963
4964 /**
4965  * i40iw_vsi_stats_init - Initialize the vsi statistics
4966  * @vsi: pointer to the vsi structure
4967  * @info: The info structure used for initialization
4968  */
4969 enum i40iw_status_code i40iw_vsi_stats_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_stats_info *info)
4970 {
4971         u8 fcn_id = info->fcn_id;
4972
4973         if (info->alloc_fcn_id)
4974                 fcn_id = i40iw_get_fcn_id(vsi->dev);
4975
4976         if (fcn_id == I40IW_INVALID_FCN_ID)
4977                 return I40IW_ERR_NOT_READY;
4978
4979         vsi->pestat = info->pestat;
4980         vsi->pestat->hw = vsi->dev->hw;
4981         vsi->pestat->vsi = vsi;
4982
4983         if (info->stats_initialize) {
4984                 i40iw_hw_stats_init(vsi->pestat, fcn_id, true);
4985                 spin_lock_init(&vsi->pestat->lock);
4986                 i40iw_hw_stats_start_timer(vsi);
4987         }
4988         vsi->stats_fcn_id_alloc = info->alloc_fcn_id;
4989         vsi->fcn_id = fcn_id;
4990         return I40IW_SUCCESS;
4991 }
4992
4993 /**
4994  * i40iw_vsi_stats_free - Free the vsi stats
4995  * @vsi: pointer to the vsi structure
4996  */
4997 void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi)
4998 {
4999         u8 fcn_id = vsi->fcn_id;
5000
5001         if (vsi->stats_fcn_id_alloc && fcn_id < I40IW_MAX_STATS_COUNT)
5002                 vsi->dev->fcn_id_array[fcn_id] = false;
5003         i40iw_hw_stats_stop_timer(vsi);
5004 }
5005
5006 static struct i40iw_cqp_ops iw_cqp_ops = {
5007         .cqp_init = i40iw_sc_cqp_init,
5008         .cqp_create = i40iw_sc_cqp_create,
5009         .cqp_post_sq = i40iw_sc_cqp_post_sq,
5010         .cqp_get_next_send_wqe = i40iw_sc_cqp_get_next_send_wqe,
5011         .cqp_destroy = i40iw_sc_cqp_destroy,
5012         .poll_for_cqp_op_done = i40iw_sc_poll_for_cqp_op_done
5013 };
5014
5015 static struct i40iw_ccq_ops iw_ccq_ops = {
5016         .ccq_init = i40iw_sc_ccq_init,
5017         .ccq_create = i40iw_sc_ccq_create,
5018         .ccq_destroy = i40iw_sc_ccq_destroy,
5019         .ccq_create_done = i40iw_sc_ccq_create_done,
5020         .ccq_get_cqe_info = i40iw_sc_ccq_get_cqe_info,
5021         .ccq_arm = i40iw_sc_ccq_arm
5022 };
5023
5024 static struct i40iw_ceq_ops iw_ceq_ops = {
5025         .ceq_init = i40iw_sc_ceq_init,
5026         .ceq_create = i40iw_sc_ceq_create,
5027         .cceq_create_done = i40iw_sc_cceq_create_done,
5028         .cceq_destroy_done = i40iw_sc_cceq_destroy_done,
5029         .cceq_create = i40iw_sc_cceq_create,
5030         .ceq_destroy = i40iw_sc_ceq_destroy,
5031         .process_ceq = i40iw_sc_process_ceq
5032 };
5033
5034 static struct i40iw_aeq_ops iw_aeq_ops = {
5035         .aeq_init = i40iw_sc_aeq_init,
5036         .aeq_create = i40iw_sc_aeq_create,
5037         .aeq_destroy = i40iw_sc_aeq_destroy,
5038         .get_next_aeqe = i40iw_sc_get_next_aeqe,
5039         .repost_aeq_entries = i40iw_sc_repost_aeq_entries,
5040         .aeq_create_done = i40iw_sc_aeq_create_done,
5041         .aeq_destroy_done = i40iw_sc_aeq_destroy_done
5042 };
5043
5044 /* iwarp pd ops */
5045 static struct i40iw_pd_ops iw_pd_ops = {
5046         .pd_init = i40iw_sc_pd_init,
5047 };
5048
5049 static struct i40iw_priv_qp_ops iw_priv_qp_ops = {
5050         .qp_init = i40iw_sc_qp_init,
5051         .qp_create = i40iw_sc_qp_create,
5052         .qp_modify = i40iw_sc_qp_modify,
5053         .qp_destroy = i40iw_sc_qp_destroy,
5054         .qp_flush_wqes = i40iw_sc_qp_flush_wqes,
5055         .qp_upload_context = i40iw_sc_qp_upload_context,
5056         .qp_setctx = i40iw_sc_qp_setctx,
5057         .qp_send_lsmm = i40iw_sc_send_lsmm,
5058         .qp_send_lsmm_nostag = i40iw_sc_send_lsmm_nostag,
5059         .qp_send_rtt = i40iw_sc_send_rtt,
5060         .qp_post_wqe0 = i40iw_sc_post_wqe0,
5061         .iw_mr_fast_register = i40iw_sc_mr_fast_register
5062 };
5063
5064 static struct i40iw_priv_cq_ops iw_priv_cq_ops = {
5065         .cq_init = i40iw_sc_cq_init,
5066         .cq_create = i40iw_sc_cq_create,
5067         .cq_destroy = i40iw_sc_cq_destroy,
5068         .cq_modify = i40iw_sc_cq_modify,
5069 };
5070
5071 static struct i40iw_mr_ops iw_mr_ops = {
5072         .alloc_stag = i40iw_sc_alloc_stag,
5073         .mr_reg_non_shared = i40iw_sc_mr_reg_non_shared,
5074         .mr_reg_shared = i40iw_sc_mr_reg_shared,
5075         .dealloc_stag = i40iw_sc_dealloc_stag,
5076         .query_stag = i40iw_sc_query_stag,
5077         .mw_alloc = i40iw_sc_mw_alloc
5078 };
5079
5080 static struct i40iw_cqp_misc_ops iw_cqp_misc_ops = {
5081         .manage_push_page = i40iw_sc_manage_push_page,
5082         .manage_hmc_pm_func_table = i40iw_sc_manage_hmc_pm_func_table,
5083         .set_hmc_resource_profile = i40iw_sc_set_hmc_resource_profile,
5084         .commit_fpm_values = i40iw_sc_commit_fpm_values,
5085         .query_fpm_values = i40iw_sc_query_fpm_values,
5086         .static_hmc_pages_allocated = i40iw_sc_static_hmc_pages_allocated,
5087         .add_arp_cache_entry = i40iw_sc_add_arp_cache_entry,
5088         .del_arp_cache_entry = i40iw_sc_del_arp_cache_entry,
5089         .query_arp_cache_entry = i40iw_sc_query_arp_cache_entry,
5090         .manage_apbvt_entry = i40iw_sc_manage_apbvt_entry,
5091         .manage_qhash_table_entry = i40iw_sc_manage_qhash_table_entry,
5092         .alloc_local_mac_ipaddr_table_entry = i40iw_sc_alloc_local_mac_ipaddr_entry,
5093         .add_local_mac_ipaddr_entry = i40iw_sc_add_local_mac_ipaddr_entry,
5094         .del_local_mac_ipaddr_entry = i40iw_sc_del_local_mac_ipaddr_entry,
5095         .cqp_nop = i40iw_sc_cqp_nop,
5096         .commit_fpm_values_done = i40iw_sc_commit_fpm_values_done,
5097         .query_fpm_values_done = i40iw_sc_query_fpm_values_done,
5098         .manage_hmc_pm_func_table_done = i40iw_sc_manage_hmc_pm_func_table_done,
5099         .update_suspend_qp = i40iw_sc_suspend_qp,
5100         .update_resume_qp = i40iw_sc_resume_qp
5101 };
5102
5103 static struct i40iw_hmc_ops iw_hmc_ops = {
5104         .init_iw_hmc = i40iw_sc_init_iw_hmc,
5105         .parse_fpm_query_buf = i40iw_sc_parse_fpm_query_buf,
5106         .configure_iw_fpm = i40iw_sc_configure_iw_fpm,
5107         .parse_fpm_commit_buf = i40iw_sc_parse_fpm_commit_buf,
5108         .create_hmc_object = i40iw_sc_create_hmc_obj,
5109         .del_hmc_object = i40iw_sc_del_hmc_obj
5110 };
5111
5112 /**
5113  * i40iw_device_init - Initialize IWARP device
5114  * @dev: IWARP device pointer
5115  * @info: IWARP init info
5116  */
5117 enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev,
5118                                          struct i40iw_device_init_info *info)
5119 {
5120         u32 val;
5121         u32 vchnl_ver = 0;
5122         u16 hmc_fcn = 0;
5123         enum i40iw_status_code ret_code = 0;
5124         u8 db_size;
5125
5126         spin_lock_init(&dev->cqp_lock);
5127
5128         i40iw_device_init_uk(&dev->dev_uk);
5129
5130         dev->debug_mask = info->debug_mask;
5131
5132         dev->hmc_fn_id = info->hmc_fn_id;
5133         dev->is_pf = info->is_pf;
5134
5135         dev->fpm_query_buf_pa = info->fpm_query_buf_pa;
5136         dev->fpm_query_buf = info->fpm_query_buf;
5137
5138         dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa;
5139         dev->fpm_commit_buf = info->fpm_commit_buf;
5140
5141         dev->hw = info->hw;
5142         dev->hw->hw_addr = info->bar0;
5143
5144         if (dev->is_pf) {
5145                 val = i40iw_rd32(dev->hw, I40E_GLPCI_DREVID);
5146                 dev->hw_rev = (u8)RS_32(val, I40E_GLPCI_DREVID_DEFAULT_REVID);
5147
5148                 val = i40iw_rd32(dev->hw, I40E_GLPCI_LBARCTRL);
5149                 db_size = (u8)RS_32(val, I40E_GLPCI_LBARCTRL_PE_DB_SIZE);
5150                 if ((db_size != I40IW_PE_DB_SIZE_4M) &&
5151                     (db_size != I40IW_PE_DB_SIZE_8M)) {
5152                         i40iw_debug(dev, I40IW_DEBUG_DEV,
5153                                     "%s: PE doorbell is not enabled in CSR val 0x%x\n",
5154                                     __func__, val);
5155                         ret_code = I40IW_ERR_PE_DOORBELL_NOT_ENABLED;
5156                         return ret_code;
5157                 }
5158                 dev->db_addr = dev->hw->hw_addr + I40IW_DB_ADDR_OFFSET;
5159                 dev->vchnl_if.vchnl_recv = i40iw_vchnl_recv_pf;
5160         } else {
5161                 dev->db_addr = dev->hw->hw_addr + I40IW_VF_DB_ADDR_OFFSET;
5162         }
5163
5164         dev->cqp_ops = &iw_cqp_ops;
5165         dev->ccq_ops = &iw_ccq_ops;
5166         dev->ceq_ops = &iw_ceq_ops;
5167         dev->aeq_ops = &iw_aeq_ops;
5168         dev->cqp_misc_ops = &iw_cqp_misc_ops;
5169         dev->iw_pd_ops = &iw_pd_ops;
5170         dev->iw_priv_qp_ops = &iw_priv_qp_ops;
5171         dev->iw_priv_cq_ops = &iw_priv_cq_ops;
5172         dev->mr_ops = &iw_mr_ops;
5173         dev->hmc_ops = &iw_hmc_ops;
5174         dev->vchnl_if.vchnl_send = info->vchnl_send;
5175         if (dev->vchnl_if.vchnl_send)
5176                 dev->vchnl_up = true;
5177         else
5178                 dev->vchnl_up = false;
5179         if (!dev->is_pf) {
5180                 dev->vchnl_if.vchnl_recv = i40iw_vchnl_recv_vf;
5181                 ret_code = i40iw_vchnl_vf_get_ver(dev, &vchnl_ver);
5182                 if (!ret_code) {
5183                         i40iw_debug(dev, I40IW_DEBUG_DEV,
5184                                     "%s: Get Channel version rc = 0x%0x, version is %u\n",
5185                                 __func__, ret_code, vchnl_ver);
5186                         ret_code = i40iw_vchnl_vf_get_hmc_fcn(dev, &hmc_fcn);
5187                         if (!ret_code) {
5188                                 i40iw_debug(dev, I40IW_DEBUG_DEV,
5189                                             "%s Get HMC function rc = 0x%0x, hmc fcn is %u\n",
5190                                             __func__, ret_code, hmc_fcn);
5191                                 dev->hmc_fn_id = (u8)hmc_fcn;
5192                         }
5193                 }
5194         }
5195         dev->iw_vf_cqp_ops = &iw_vf_cqp_ops;
5196
5197         return ret_code;
5198 }