drivers/net/ethernet/amazon/ena/ena_com.c

   1 /*
   2  * Copyright 2015 Amazon.com, Inc. or its affiliates.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include "ena_com.h"
  34
  35 /*****************************************************************************/
  36 /*****************************************************************************/
  37
  38 /* Timeout in micro-sec */
  39 #define ADMIN_CMD_TIMEOUT_US (3000000)
  40
  41 #define ENA_ASYNC_QUEUE_DEPTH 16
  42 #define ENA_ADMIN_QUEUE_DEPTH 32
  43
  44 #define MIN_ENA_VER (((ENA_COMMON_SPEC_VERSION_MAJOR) << \
  45                 ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) \
  46                 | (ENA_COMMON_SPEC_VERSION_MINOR))
  47
  48 #define ENA_CTRL_MAJOR          0
  49 #define ENA_CTRL_MINOR          0
  50 #define ENA_CTRL_SUB_MINOR      1
  51
  52 #define MIN_ENA_CTRL_VER \
  53         (((ENA_CTRL_MAJOR) << \
  54         (ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT)) | \
  55         ((ENA_CTRL_MINOR) << \
  56         (ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT)) | \
  57         (ENA_CTRL_SUB_MINOR))
  58
  59 #define ENA_DMA_ADDR_TO_UINT32_LOW(x)   ((u32)((u64)(x)))
  60 #define ENA_DMA_ADDR_TO_UINT32_HIGH(x)  ((u32)(((u64)(x)) >> 32))
  61
  62 #define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF
  63
  64 #define ENA_REGS_ADMIN_INTR_MASK 1
  65
  66 #define ENA_POLL_MS     5
  67
  68 /*****************************************************************************/
  69 /*****************************************************************************/
  70 /*****************************************************************************/
  71
  72 enum ena_cmd_status {
  73         ENA_CMD_SUBMITTED,
  74         ENA_CMD_COMPLETED,
  75         /* Abort - canceled by the driver */
  76         ENA_CMD_ABORTED,
  77 };
  78
  79 struct ena_comp_ctx {
  80         struct completion wait_event;
  81         struct ena_admin_acq_entry *user_cqe;
  82         u32 comp_size;
  83         enum ena_cmd_status status;
  84         /* status from the device */
  85         u8 comp_status;
  86         u8 cmd_opcode;
  87         bool occupied;
  88 };
  89
  90 struct ena_com_stats_ctx {
  91         struct ena_admin_aq_get_stats_cmd get_cmd;
  92         struct ena_admin_acq_get_stats_resp get_resp;
  93 };
  94
  95 static inline int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
  96                                        struct ena_common_mem_addr *ena_addr,
  97                                        dma_addr_t addr)
  98 {
  99         if ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr) {
 100                 pr_err("dma address has more bits that the device supports\n");
 101                 return -EINVAL;
 102         }
 103
 104         ena_addr->mem_addr_low = lower_32_bits(addr);
 105         ena_addr->mem_addr_high = (u16)upper_32_bits(addr);
 106
 107         return 0;
 108 }
 109
 110 static int ena_com_admin_init_sq(struct ena_com_admin_queue *queue)
 111 {
 112         struct ena_com_admin_sq *sq = &queue->sq;
 113         u16 size = ADMIN_SQ_SIZE(queue->q_depth);
 114
 115         sq->entries = dma_zalloc_coherent(queue->q_dmadev, size, &sq->dma_addr,
 116                                           GFP_KERNEL);
 117
 118         if (!sq->entries) {
 119                 pr_err("memory allocation failed");
 120                 return -ENOMEM;
 121         }
 122
 123         sq->head = 0;
 124         sq->tail = 0;
 125         sq->phase = 1;
 126
 127         sq->db_addr = NULL;
 128
 129         return 0;
 130 }
 131
 132 static int ena_com_admin_init_cq(struct ena_com_admin_queue *queue)
 133 {
 134         struct ena_com_admin_cq *cq = &queue->cq;
 135         u16 size = ADMIN_CQ_SIZE(queue->q_depth);
 136
 137         cq->entries = dma_zalloc_coherent(queue->q_dmadev, size, &cq->dma_addr,
 138                                           GFP_KERNEL);
 139
 140         if (!cq->entries) {
 141                 pr_err("memory allocation failed");
 142                 return -ENOMEM;
 143         }
 144
 145         cq->head = 0;
 146         cq->phase = 1;
 147
 148         return 0;
 149 }
 150
 151 static int ena_com_admin_init_aenq(struct ena_com_dev *dev,
 152                                    struct ena_aenq_handlers *aenq_handlers)
 153 {
 154         struct ena_com_aenq *aenq = &dev->aenq;
 155         u32 addr_low, addr_high, aenq_caps;
 156         u16 size;
 157
 158         dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH;
 159         size = ADMIN_AENQ_SIZE(ENA_ASYNC_QUEUE_DEPTH);
 160         aenq->entries = dma_zalloc_coherent(dev->dmadev, size, &aenq->dma_addr,
 161                                             GFP_KERNEL);
 162
 163         if (!aenq->entries) {
 164                 pr_err("memory allocation failed");
 165                 return -ENOMEM;
 166         }
 167
 168         aenq->head = aenq->q_depth;
 169         aenq->phase = 1;
 170
 171         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(aenq->dma_addr);
 172         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(aenq->dma_addr);
 173
 174         writel(addr_low, dev->reg_bar + ENA_REGS_AENQ_BASE_LO_OFF);
 175         writel(addr_high, dev->reg_bar + ENA_REGS_AENQ_BASE_HI_OFF);
 176
 177         aenq_caps = 0;
 178         aenq_caps |= dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK;
 179         aenq_caps |= (sizeof(struct ena_admin_aenq_entry)
 180                       << ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) &
 181                      ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK;
 182         writel(aenq_caps, dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF);
 183
 184         if (unlikely(!aenq_handlers)) {
 185                 pr_err("aenq handlers pointer is NULL\n");
 186                 return -EINVAL;
 187         }
 188
 189         aenq->aenq_handlers = aenq_handlers;
 190
 191         return 0;
 192 }
 193
 194 static inline void comp_ctxt_release(struct ena_com_admin_queue *queue,
 195                                      struct ena_comp_ctx *comp_ctx)
 196 {
 197         comp_ctx->occupied = false;
 198         atomic_dec(&queue->outstanding_cmds);
 199 }
 200
 201 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *queue,
 202                                           u16 command_id, bool capture)
 203 {
 204         if (unlikely(command_id >= queue->q_depth)) {
 205                 pr_err("command id is larger than the queue size. cmd_id: %u queue size %d\n",
 206                        command_id, queue->q_depth);
 207                 return NULL;
 208         }
 209
 210         if (unlikely(queue->comp_ctx[command_id].occupied && capture)) {
 211                 pr_err("Completion context is occupied\n");
 212                 return NULL;
 213         }
 214
 215         if (capture) {
 216                 atomic_inc(&queue->outstanding_cmds);
 217                 queue->comp_ctx[command_id].occupied = true;
 218         }
 219
 220         return &queue->comp_ctx[command_id];
 221 }
 222
 223 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
 224                                                        struct ena_admin_aq_entry *cmd,
 225                                                        size_t cmd_size_in_bytes,
 226                                                        struct ena_admin_acq_entry *comp,
 227                                                        size_t comp_size_in_bytes)
 228 {
 229         struct ena_comp_ctx *comp_ctx;
 230         u16 tail_masked, cmd_id;
 231         u16 queue_size_mask;
 232         u16 cnt;
 233
 234         queue_size_mask = admin_queue->q_depth - 1;
 235
 236         tail_masked = admin_queue->sq.tail & queue_size_mask;
 237
 238         /* In case of queue FULL */
 239         cnt = atomic_read(&admin_queue->outstanding_cmds);
 240         if (cnt >= admin_queue->q_depth) {
 241                 pr_debug("admin queue is full.\n");
 242                 admin_queue->stats.out_of_space++;
 243                 return ERR_PTR(-ENOSPC);
 244         }
 245
 246         cmd_id = admin_queue->curr_cmd_id;
 247
 248         cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &
 249                 ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK;
 250
 251         cmd->aq_common_descriptor.command_id |= cmd_id &
 252                 ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
 253
 254         comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);
 255         if (unlikely(!comp_ctx))
 256                 return ERR_PTR(-EINVAL);
 257
 258         comp_ctx->status = ENA_CMD_SUBMITTED;
 259         comp_ctx->comp_size = (u32)comp_size_in_bytes;
 260         comp_ctx->user_cqe = comp;
 261         comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode;
 262
 263         reinit_completion(&comp_ctx->wait_event);
 264
 265         memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);
 266
 267         admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &
 268                 queue_size_mask;
 269
 270         admin_queue->sq.tail++;
 271         admin_queue->stats.submitted_cmd++;
 272
 273         if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))
 274                 admin_queue->sq.phase = !admin_queue->sq.phase;
 275
 276         writel(admin_queue->sq.tail, admin_queue->sq.db_addr);
 277
 278         return comp_ctx;
 279 }
 280
 281 static inline int ena_com_init_comp_ctxt(struct ena_com_admin_queue *queue)
 282 {
 283         size_t size = queue->q_depth * sizeof(struct ena_comp_ctx);
 284         struct ena_comp_ctx *comp_ctx;
 285         u16 i;
 286
 287         queue->comp_ctx = devm_kzalloc(queue->q_dmadev, size, GFP_KERNEL);
 288         if (unlikely(!queue->comp_ctx)) {
 289                 pr_err("memory allocation failed");
 290                 return -ENOMEM;
 291         }
 292
 293         for (i = 0; i < queue->q_depth; i++) {
 294                 comp_ctx = get_comp_ctxt(queue, i, false);
 295                 if (comp_ctx)
 296                         init_completion(&comp_ctx->wait_event);
 297         }
 298
 299         return 0;
 300 }
 301
 302 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
 303                                                      struct ena_admin_aq_entry *cmd,
 304                                                      size_t cmd_size_in_bytes,
 305                                                      struct ena_admin_acq_entry *comp,
 306                                                      size_t comp_size_in_bytes)
 307 {
 308         unsigned long flags;
 309         struct ena_comp_ctx *comp_ctx;
 310
 311         spin_lock_irqsave(&admin_queue->q_lock, flags);
 312         if (unlikely(!admin_queue->running_state)) {
 313                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
 314                 return ERR_PTR(-ENODEV);
 315         }
 316         comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,
 317                                               cmd_size_in_bytes,
 318                                               comp,
 319                                               comp_size_in_bytes);
 320         if (IS_ERR(comp_ctx))
 321                 admin_queue->running_state = false;
 322         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
 323
 324         return comp_ctx;
 325 }
 326
 327 static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
 328                               struct ena_com_create_io_ctx *ctx,
 329                               struct ena_com_io_sq *io_sq)
 330 {
 331         size_t size;
 332         int dev_node = 0;
 333
 334         memset(&io_sq->desc_addr, 0x0, sizeof(io_sq->desc_addr));
 335
 336         io_sq->desc_entry_size =
 337                 (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
 338                 sizeof(struct ena_eth_io_tx_desc) :
 339                 sizeof(struct ena_eth_io_rx_desc);
 340
 341         size = io_sq->desc_entry_size * io_sq->q_depth;
 342
 343         if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) {
 344                 dev_node = dev_to_node(ena_dev->dmadev);
 345                 set_dev_node(ena_dev->dmadev, ctx->numa_node);
 346                 io_sq->desc_addr.virt_addr =
 347                         dma_zalloc_coherent(ena_dev->dmadev, size,
 348                                             &io_sq->desc_addr.phys_addr,
 349                                             GFP_KERNEL);
 350                 set_dev_node(ena_dev->dmadev, dev_node);
 351                 if (!io_sq->desc_addr.virt_addr) {
 352                         io_sq->desc_addr.virt_addr =
 353                                 dma_zalloc_coherent(ena_dev->dmadev, size,
 354                                                     &io_sq->desc_addr.phys_addr,
 355                                                     GFP_KERNEL);
 356                 }
 357         } else {
 358                 dev_node = dev_to_node(ena_dev->dmadev);
 359                 set_dev_node(ena_dev->dmadev, ctx->numa_node);
 360                 io_sq->desc_addr.virt_addr =
 361                         devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
 362                 set_dev_node(ena_dev->dmadev, dev_node);
 363                 if (!io_sq->desc_addr.virt_addr) {
 364                         io_sq->desc_addr.virt_addr =
 365                                 devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
 366                 }
 367         }
 368
 369         if (!io_sq->desc_addr.virt_addr) {
 370                 pr_err("memory allocation failed");
 371                 return -ENOMEM;
 372         }
 373
 374         io_sq->tail = 0;
 375         io_sq->next_to_comp = 0;
 376         io_sq->phase = 1;
 377
 378         return 0;
 379 }
 380
 381 static int ena_com_init_io_cq(struct ena_com_dev *ena_dev,
 382                               struct ena_com_create_io_ctx *ctx,
 383                               struct ena_com_io_cq *io_cq)
 384 {
 385         size_t size;
 386         int prev_node = 0;
 387
 388         memset(&io_cq->cdesc_addr, 0x0, sizeof(io_cq->cdesc_addr));
 389
 390         /* Use the basic completion descriptor for Rx */
 391         io_cq->cdesc_entry_size_in_bytes =
 392                 (io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
 393                 sizeof(struct ena_eth_io_tx_cdesc) :
 394                 sizeof(struct ena_eth_io_rx_cdesc_base);
 395
 396         size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
 397
 398         prev_node = dev_to_node(ena_dev->dmadev);
 399         set_dev_node(ena_dev->dmadev, ctx->numa_node);
 400         io_cq->cdesc_addr.virt_addr =
 401                 dma_zalloc_coherent(ena_dev->dmadev, size,
 402                                     &io_cq->cdesc_addr.phys_addr, GFP_KERNEL);
 403         set_dev_node(ena_dev->dmadev, prev_node);
 404         if (!io_cq->cdesc_addr.virt_addr) {
 405                 io_cq->cdesc_addr.virt_addr =
 406                         dma_zalloc_coherent(ena_dev->dmadev, size,
 407                                             &io_cq->cdesc_addr.phys_addr,
 408                                             GFP_KERNEL);
 409         }
 410
 411         if (!io_cq->cdesc_addr.virt_addr) {
 412                 pr_err("memory allocation failed");
 413                 return -ENOMEM;
 414         }
 415
 416         io_cq->phase = 1;
 417         io_cq->head = 0;
 418
 419         return 0;
 420 }
 421
 422 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,
 423                                                    struct ena_admin_acq_entry *cqe)
 424 {
 425         struct ena_comp_ctx *comp_ctx;
 426         u16 cmd_id;
 427
 428         cmd_id = cqe->acq_common_descriptor.command &
 429                 ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;
 430
 431         comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);
 432         if (unlikely(!comp_ctx)) {
 433                 pr_err("comp_ctx is NULL. Changing the admin queue running state\n");
 434                 admin_queue->running_state = false;
 435                 return;
 436         }
 437
 438         comp_ctx->status = ENA_CMD_COMPLETED;
 439         comp_ctx->comp_status = cqe->acq_common_descriptor.status;
 440
 441         if (comp_ctx->user_cqe)
 442                 memcpy(comp_ctx->user_cqe, (void *)cqe, comp_ctx->comp_size);
 443
 444         if (!admin_queue->polling)
 445                 complete(&comp_ctx->wait_event);
 446 }
 447
 448 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)
 449 {
 450         struct ena_admin_acq_entry *cqe = NULL;
 451         u16 comp_num = 0;
 452         u16 head_masked;
 453         u8 phase;
 454
 455         head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);
 456         phase = admin_queue->cq.phase;
 457
 458         cqe = &admin_queue->cq.entries[head_masked];
 459
 460         /* Go over all the completions */
 461         while ((cqe->acq_common_descriptor.flags &
 462                         ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) {
 463                 /* Do not read the rest of the completion entry before the
 464                  * phase bit was validated
 465                  */
 466                 rmb();
 467                 ena_com_handle_single_admin_completion(admin_queue, cqe);
 468
 469                 head_masked++;
 470                 comp_num++;
 471                 if (unlikely(head_masked == admin_queue->q_depth)) {
 472                         head_masked = 0;
 473                         phase = !phase;
 474                 }
 475
 476                 cqe = &admin_queue->cq.entries[head_masked];
 477         }
 478
 479         admin_queue->cq.head += comp_num;
 480         admin_queue->cq.phase = phase;
 481         admin_queue->sq.head += comp_num;
 482         admin_queue->stats.completed_cmd += comp_num;
 483 }
 484
 485 static int ena_com_comp_status_to_errno(u8 comp_status)
 486 {
 487         if (unlikely(comp_status != 0))
 488                 pr_err("admin command failed[%u]\n", comp_status);
 489
 490         if (unlikely(comp_status > ENA_ADMIN_UNKNOWN_ERROR))
 491                 return -EINVAL;
 492
 493         switch (comp_status) {
 494         case ENA_ADMIN_SUCCESS:
 495                 return 0;
 496         case ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE:
 497                 return -ENOMEM;
 498         case ENA_ADMIN_UNSUPPORTED_OPCODE:
 499                 return -EOPNOTSUPP;
 500         case ENA_ADMIN_BAD_OPCODE:
 501         case ENA_ADMIN_MALFORMED_REQUEST:
 502         case ENA_ADMIN_ILLEGAL_PARAMETER:
 503         case ENA_ADMIN_UNKNOWN_ERROR:
 504                 return -EINVAL;
 505         }
 506
 507         return 0;
 508 }
 509
 510 static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_ctx,
 511                                                      struct ena_com_admin_queue *admin_queue)
 512 {
 513         unsigned long flags, timeout;
 514         int ret;
 515
 516         timeout = jiffies + usecs_to_jiffies(admin_queue->completion_timeout);
 517
 518         while (1) {
 519                 spin_lock_irqsave(&admin_queue->q_lock, flags);
 520                 ena_com_handle_admin_completion(admin_queue);
 521                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
 522
 523                 if (comp_ctx->status != ENA_CMD_SUBMITTED)
 524                         break;
 525
 526                 if (time_is_before_jiffies(timeout)) {
 527                         pr_err("Wait for completion (polling) timeout\n");
 528                         /* ENA didn't have any completion */
 529                         spin_lock_irqsave(&admin_queue->q_lock, flags);
 530                         admin_queue->stats.no_completion++;
 531                         admin_queue->running_state = false;
 532                         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
 533
 534                         ret = -ETIME;
 535                         goto err;
 536                 }
 537
 538                 msleep(ENA_POLL_MS);
 539         }
 540
 541         if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) {
 542                 pr_err("Command was aborted\n");
 543                 spin_lock_irqsave(&admin_queue->q_lock, flags);
 544                 admin_queue->stats.aborted_cmd++;
 545                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
 546                 ret = -ENODEV;
 547                 goto err;
 548         }
 549
 550         WARN(comp_ctx->status != ENA_CMD_COMPLETED, "Invalid comp status %d\n",
 551              comp_ctx->status);
 552
 553         ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
 554 err:
 555         comp_ctxt_release(admin_queue, comp_ctx);
 556         return ret;
 557 }
 558
 559 static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx,
 560                                                         struct ena_com_admin_queue *admin_queue)
 561 {
 562         unsigned long flags;
 563         int ret;
 564
 565         wait_for_completion_timeout(&comp_ctx->wait_event,
 566                                     usecs_to_jiffies(
 567                                             admin_queue->completion_timeout));
 568
 569         /* In case the command wasn't completed find out the root cause.
 570          * There might be 2 kinds of errors
 571          * 1) No completion (timeout reached)
 572          * 2) There is completion but the device didn't get any msi-x interrupt.
 573          */
 574         if (unlikely(comp_ctx->status == ENA_CMD_SUBMITTED)) {
 575                 spin_lock_irqsave(&admin_queue->q_lock, flags);
 576                 ena_com_handle_admin_completion(admin_queue);
 577                 admin_queue->stats.no_completion++;
 578                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
 579
 580                 if (comp_ctx->status == ENA_CMD_COMPLETED)
 581                         pr_err("The ena device have completion but the driver didn't receive any MSI-X interrupt (cmd %d)\n",
 582                                comp_ctx->cmd_opcode);
 583                 else
 584                         pr_err("The ena device doesn't send any completion for the admin cmd %d status %d\n",
 585                                comp_ctx->cmd_opcode, comp_ctx->status);
 586
 587                 admin_queue->running_state = false;
 588                 ret = -ETIME;
 589                 goto err;
 590         }
 591
 592         ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
 593 err:
 594         comp_ctxt_release(admin_queue, comp_ctx);
 595         return ret;
 596 }
 597
 598 /* This method read the hardware device register through posting writes
 599  * and waiting for response
 600  * On timeout the function will return ENA_MMIO_READ_TIMEOUT
 601  */
 602 static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
 603 {
 604         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
 605         volatile struct ena_admin_ena_mmio_req_read_less_resp *read_resp =
 606                 mmio_read->read_resp;
 607         u32 mmio_read_reg, ret, i;
 608         unsigned long flags;
 609         u32 timeout = mmio_read->reg_read_to;
 610
 611         might_sleep();
 612
 613         if (timeout == 0)
 614                 timeout = ENA_REG_READ_TIMEOUT;
 615
 616         /* If readless is disabled, perform regular read */
 617         if (!mmio_read->readless_supported)
 618                 return readl(ena_dev->reg_bar + offset);
 619
 620         spin_lock_irqsave(&mmio_read->lock, flags);
 621         mmio_read->seq_num++;
 622
 623         read_resp->req_id = mmio_read->seq_num + 0xDEAD;
 624         mmio_read_reg = (offset << ENA_REGS_MMIO_REG_READ_REG_OFF_SHIFT) &
 625                         ENA_REGS_MMIO_REG_READ_REG_OFF_MASK;
 626         mmio_read_reg |= mmio_read->seq_num &
 627                         ENA_REGS_MMIO_REG_READ_REQ_ID_MASK;
 628
 629         /* make sure read_resp->req_id get updated before the hw can write
 630          * there
 631          */
 632         wmb();
 633
 634         writel_relaxed(mmio_read_reg,
 635                        ena_dev->reg_bar + ENA_REGS_MMIO_REG_READ_OFF);
 636
 637         mmiowb();
 638         for (i = 0; i < timeout; i++) {
 639                 if (read_resp->req_id == mmio_read->seq_num)
 640                         break;
 641
 642                 udelay(1);
 643         }
 644
 645         if (unlikely(i == timeout)) {
 646                 pr_err("reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n",
 647                        mmio_read->seq_num, offset, read_resp->req_id,
 648                        read_resp->reg_off);
 649                 ret = ENA_MMIO_READ_TIMEOUT;
 650                 goto err;
 651         }
 652
 653         if (read_resp->reg_off != offset) {
 654                 pr_err("Read failure: wrong offset provided");
 655                 ret = ENA_MMIO_READ_TIMEOUT;
 656         } else {
 657                 ret = read_resp->reg_val;
 658         }
 659 err:
 660         spin_unlock_irqrestore(&mmio_read->lock, flags);
 661
 662         return ret;
 663 }
 664
 665 /* There are two types to wait for completion.
 666  * Polling mode - wait until the completion is available.
 667  * Async mode - wait on wait queue until the completion is ready
 668  * (or the timeout expired).
 669  * It is expected that the IRQ called ena_com_handle_admin_completion
 670  * to mark the completions.
 671  */
 672 static int ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx,
 673                                              struct ena_com_admin_queue *admin_queue)
 674 {
 675         if (admin_queue->polling)
 676                 return ena_com_wait_and_process_admin_cq_polling(comp_ctx,
 677                                                                  admin_queue);
 678
 679         return ena_com_wait_and_process_admin_cq_interrupts(comp_ctx,
 680                                                             admin_queue);
 681 }
 682
 683 static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev,
 684                                  struct ena_com_io_sq *io_sq)
 685 {
 686         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
 687         struct ena_admin_aq_destroy_sq_cmd destroy_cmd;
 688         struct ena_admin_acq_destroy_sq_resp_desc destroy_resp;
 689         u8 direction;
 690         int ret;
 691
 692         memset(&destroy_cmd, 0x0, sizeof(destroy_cmd));
 693
 694         if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
 695                 direction = ENA_ADMIN_SQ_DIRECTION_TX;
 696         else
 697                 direction = ENA_ADMIN_SQ_DIRECTION_RX;
 698
 699         destroy_cmd.sq.sq_identity |= (direction <<
 700                 ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) &
 701                 ENA_ADMIN_SQ_SQ_DIRECTION_MASK;
 702
 703         destroy_cmd.sq.sq_idx = io_sq->idx;
 704         destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_SQ;
 705
 706         ret = ena_com_execute_admin_command(admin_queue,
 707                                             (struct ena_admin_aq_entry *)&destroy_cmd,
 708                                             sizeof(destroy_cmd),
 709                                             (struct ena_admin_acq_entry *)&destroy_resp,
 710                                             sizeof(destroy_resp));
 711
 712         if (unlikely(ret && (ret != -ENODEV)))
 713                 pr_err("failed to destroy io sq error: %d\n", ret);
 714
 715         return ret;
 716 }
 717
 718 static void ena_com_io_queue_free(struct ena_com_dev *ena_dev,
 719                                   struct ena_com_io_sq *io_sq,
 720                                   struct ena_com_io_cq *io_cq)
 721 {
 722         size_t size;
 723
 724         if (io_cq->cdesc_addr.virt_addr) {
 725                 size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
 726
 727                 dma_free_coherent(ena_dev->dmadev, size,
 728                                   io_cq->cdesc_addr.virt_addr,
 729                                   io_cq->cdesc_addr.phys_addr);
 730
 731                 io_cq->cdesc_addr.virt_addr = NULL;
 732         }
 733
 734         if (io_sq->desc_addr.virt_addr) {
 735                 size = io_sq->desc_entry_size * io_sq->q_depth;
 736
 737                 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
 738                         dma_free_coherent(ena_dev->dmadev, size,
 739                                           io_sq->desc_addr.virt_addr,
 740                                           io_sq->desc_addr.phys_addr);
 741                 else
 742                         devm_kfree(ena_dev->dmadev, io_sq->desc_addr.virt_addr);
 743
 744                 io_sq->desc_addr.virt_addr = NULL;
 745         }
 746 }
 747
 748 static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout,
 749                                 u16 exp_state)
 750 {
 751         u32 val, i;
 752
 753         /* Convert timeout from resolution of 100ms to ENA_POLL_MS */
 754         timeout = (timeout * 100) / ENA_POLL_MS;
 755
 756         for (i = 0; i < timeout; i++) {
 757                 val = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
 758
 759                 if (unlikely(val == ENA_MMIO_READ_TIMEOUT)) {
 760                         pr_err("Reg read timeout occurred\n");
 761                         return -ETIME;
 762                 }
 763
 764                 if ((val & ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK) ==
 765                         exp_state)
 766                         return 0;
 767
 768                 msleep(ENA_POLL_MS);
 769         }
 770
 771         return -ETIME;
 772 }
 773
 774 static bool ena_com_check_supported_feature_id(struct ena_com_dev *ena_dev,
 775                                                enum ena_admin_aq_feature_id feature_id)
 776 {
 777         u32 feature_mask = 1 << feature_id;
 778
 779         /* Device attributes is always supported */
 780         if ((feature_id != ENA_ADMIN_DEVICE_ATTRIBUTES) &&
 781             !(ena_dev->supported_features & feature_mask))
 782                 return false;
 783
 784         return true;
 785 }
 786
 787 static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev,
 788                                   struct ena_admin_get_feat_resp *get_resp,
 789                                   enum ena_admin_aq_feature_id feature_id,
 790                                   dma_addr_t control_buf_dma_addr,
 791                                   u32 control_buff_size)
 792 {
 793         struct ena_com_admin_queue *admin_queue;
 794         struct ena_admin_get_feat_cmd get_cmd;
 795         int ret;
 796
 797         if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) {
 798                 pr_debug("Feature %d isn't supported\n", feature_id);
 799                 return -EOPNOTSUPP;
 800         }
 801
 802         memset(&get_cmd, 0x0, sizeof(get_cmd));
 803         admin_queue = &ena_dev->admin_queue;
 804
 805         get_cmd.aq_common_descriptor.opcode = ENA_ADMIN_GET_FEATURE;
 806
 807         if (control_buff_size)
 808                 get_cmd.aq_common_descriptor.flags =
 809                         ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
 810         else
 811                 get_cmd.aq_common_descriptor.flags = 0;
 812
 813         ret = ena_com_mem_addr_set(ena_dev,
 814                                    &get_cmd.control_buffer.address,
 815                                    control_buf_dma_addr);
 816         if (unlikely(ret)) {
 817                 pr_err("memory address set failed\n");
 818                 return ret;
 819         }
 820
 821         get_cmd.control_buffer.length = control_buff_size;
 822
 823         get_cmd.feat_common.feature_id = feature_id;
 824
 825         ret = ena_com_execute_admin_command(admin_queue,
 826                                             (struct ena_admin_aq_entry *)
 827                                             &get_cmd,
 828                                             sizeof(get_cmd),
 829                                             (struct ena_admin_acq_entry *)
 830                                             get_resp,
 831                                             sizeof(*get_resp));
 832
 833         if (unlikely(ret))
 834                 pr_err("Failed to submit get_feature command %d error: %d\n",
 835                        feature_id, ret);
 836
 837         return ret;
 838 }
 839
 840 static int ena_com_get_feature(struct ena_com_dev *ena_dev,
 841                                struct ena_admin_get_feat_resp *get_resp,
 842                                enum ena_admin_aq_feature_id feature_id)
 843 {
 844         return ena_com_get_feature_ex(ena_dev,
 845                                       get_resp,
 846                                       feature_id,
 847                                       0,
 848                                       0);
 849 }
 850
 851 static int ena_com_hash_key_allocate(struct ena_com_dev *ena_dev)
 852 {
 853         struct ena_rss *rss = &ena_dev->rss;
 854
 855         rss->hash_key =
 856                 dma_zalloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_key),
 857                                     &rss->hash_key_dma_addr, GFP_KERNEL);
 858
 859         if (unlikely(!rss->hash_key))
 860                 return -ENOMEM;
 861
 862         return 0;
 863 }
 864
 865 static void ena_com_hash_key_destroy(struct ena_com_dev *ena_dev)
 866 {
 867         struct ena_rss *rss = &ena_dev->rss;
 868
 869         if (rss->hash_key)
 870                 dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_key),
 871                                   rss->hash_key, rss->hash_key_dma_addr);
 872         rss->hash_key = NULL;
 873 }
 874
 875 static int ena_com_hash_ctrl_init(struct ena_com_dev *ena_dev)
 876 {
 877         struct ena_rss *rss = &ena_dev->rss;
 878
 879         rss->hash_ctrl =
 880                 dma_zalloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl),
 881                                     &rss->hash_ctrl_dma_addr, GFP_KERNEL);
 882
 883         if (unlikely(!rss->hash_ctrl))
 884                 return -ENOMEM;
 885
 886         return 0;
 887 }
 888
 889 static void ena_com_hash_ctrl_destroy(struct ena_com_dev *ena_dev)
 890 {
 891         struct ena_rss *rss = &ena_dev->rss;
 892
 893         if (rss->hash_ctrl)
 894                 dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl),
 895                                   rss->hash_ctrl, rss->hash_ctrl_dma_addr);
 896         rss->hash_ctrl = NULL;
 897 }
 898
 899 static int ena_com_indirect_table_allocate(struct ena_com_dev *ena_dev,
 900                                            u16 log_size)
 901 {
 902         struct ena_rss *rss = &ena_dev->rss;
 903         struct ena_admin_get_feat_resp get_resp;
 904         size_t tbl_size;
 905         int ret;
 906
 907         ret = ena_com_get_feature(ena_dev, &get_resp,
 908                                   ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
 909         if (unlikely(ret))
 910                 return ret;
 911
 912         if ((get_resp.u.ind_table.min_size > log_size) ||
 913             (get_resp.u.ind_table.max_size < log_size)) {
 914                 pr_err("indirect table size doesn't fit. requested size: %d while min is:%d and max %d\n",
 915                        1 << log_size, 1 << get_resp.u.ind_table.min_size,
 916                        1 << get_resp.u.ind_table.max_size);
 917                 return -EINVAL;
 918         }
 919
 920         tbl_size = (1ULL << log_size) *
 921                 sizeof(struct ena_admin_rss_ind_table_entry);
 922
 923         rss->rss_ind_tbl =
 924                 dma_zalloc_coherent(ena_dev->dmadev, tbl_size,
 925                                     &rss->rss_ind_tbl_dma_addr, GFP_KERNEL);
 926         if (unlikely(!rss->rss_ind_tbl))
 927                 goto mem_err1;
 928
 929         tbl_size = (1ULL << log_size) * sizeof(u16);
 930         rss->host_rss_ind_tbl =
 931                 devm_kzalloc(ena_dev->dmadev, tbl_size, GFP_KERNEL);
 932         if (unlikely(!rss->host_rss_ind_tbl))
 933                 goto mem_err2;
 934
 935         rss->tbl_log_size = log_size;
 936
 937         return 0;
 938
 939 mem_err2:
 940         tbl_size = (1ULL << log_size) *
 941                 sizeof(struct ena_admin_rss_ind_table_entry);
 942
 943         dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl,
 944                           rss->rss_ind_tbl_dma_addr);
 945         rss->rss_ind_tbl = NULL;
 946 mem_err1:
 947         rss->tbl_log_size = 0;
 948         return -ENOMEM;
 949 }
 950
 951 static void ena_com_indirect_table_destroy(struct ena_com_dev *ena_dev)
 952 {
 953         struct ena_rss *rss = &ena_dev->rss;
 954         size_t tbl_size = (1ULL << rss->tbl_log_size) *
 955                 sizeof(struct ena_admin_rss_ind_table_entry);
 956
 957         if (rss->rss_ind_tbl)
 958                 dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl,
 959                                   rss->rss_ind_tbl_dma_addr);
 960         rss->rss_ind_tbl = NULL;
 961
 962         if (rss->host_rss_ind_tbl)
 963                 devm_kfree(ena_dev->dmadev, rss->host_rss_ind_tbl);
 964         rss->host_rss_ind_tbl = NULL;
 965 }
 966
 967 static int ena_com_create_io_sq(struct ena_com_dev *ena_dev,
 968                                 struct ena_com_io_sq *io_sq, u16 cq_idx)
 969 {
 970         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
 971         struct ena_admin_aq_create_sq_cmd create_cmd;
 972         struct ena_admin_acq_create_sq_resp_desc cmd_completion;
 973         u8 direction;
 974         int ret;
 975
 976         memset(&create_cmd, 0x0, sizeof(create_cmd));
 977
 978         create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_SQ;
 979
 980         if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
 981                 direction = ENA_ADMIN_SQ_DIRECTION_TX;
 982         else
 983                 direction = ENA_ADMIN_SQ_DIRECTION_RX;
 984
 985         create_cmd.sq_identity |= (direction <<
 986                 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT) &
 987                 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK;
 988
 989         create_cmd.sq_caps_2 |= io_sq->mem_queue_type &
 990                 ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK;
 991
 992         create_cmd.sq_caps_2 |= (ENA_ADMIN_COMPLETION_POLICY_DESC <<
 993                 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT) &
 994                 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK;
 995
 996         create_cmd.sq_caps_3 |=
 997                 ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK;
 998
 999         create_cmd.cq_idx = cq_idx;
1000         create_cmd.sq_depth = io_sq->q_depth;
1001
1002         if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) {
1003                 ret = ena_com_mem_addr_set(ena_dev,
1004                                            &create_cmd.sq_ba,
1005                                            io_sq->desc_addr.phys_addr);
1006                 if (unlikely(ret)) {
1007                         pr_err("memory address set failed\n");
1008                         return ret;
1009                 }
1010         }
1011
1012         ret = ena_com_execute_admin_command(admin_queue,
1013                                             (struct ena_admin_aq_entry *)&create_cmd,
1014                                             sizeof(create_cmd),
1015                                             (struct ena_admin_acq_entry *)&cmd_completion,
1016                                             sizeof(cmd_completion));
1017         if (unlikely(ret)) {
1018                 pr_err("Failed to create IO SQ. error: %d\n", ret);
1019                 return ret;
1020         }
1021
1022         io_sq->idx = cmd_completion.sq_idx;
1023
1024         io_sq->db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1025                 (uintptr_t)cmd_completion.sq_doorbell_offset);
1026
1027         if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
1028                 io_sq->header_addr = (u8 __iomem *)((uintptr_t)ena_dev->mem_bar
1029                                 + cmd_completion.llq_headers_offset);
1030
1031                 io_sq->desc_addr.pbuf_dev_addr =
1032                         (u8 __iomem *)((uintptr_t)ena_dev->mem_bar +
1033                         cmd_completion.llq_descriptors_offset);
1034         }
1035
1036         pr_debug("created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth);
1037
1038         return ret;
1039 }
1040
1041 static int ena_com_ind_tbl_convert_to_device(struct ena_com_dev *ena_dev)
1042 {
1043         struct ena_rss *rss = &ena_dev->rss;
1044         struct ena_com_io_sq *io_sq;
1045         u16 qid;
1046         int i;
1047
1048         for (i = 0; i < 1 << rss->tbl_log_size; i++) {
1049                 qid = rss->host_rss_ind_tbl[i];
1050                 if (qid >= ENA_TOTAL_NUM_QUEUES)
1051                         return -EINVAL;
1052
1053                 io_sq = &ena_dev->io_sq_queues[qid];
1054
1055                 if (io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX)
1056                         return -EINVAL;
1057
1058                 rss->rss_ind_tbl[i].cq_idx = io_sq->idx;
1059         }
1060
1061         return 0;
1062 }
1063
1064 static int ena_com_ind_tbl_convert_from_device(struct ena_com_dev *ena_dev)
1065 {
1066         u16 dev_idx_to_host_tbl[ENA_TOTAL_NUM_QUEUES] = { (u16)-1 };
1067         struct ena_rss *rss = &ena_dev->rss;
1068         u8 idx;
1069         u16 i;
1070
1071         for (i = 0; i < ENA_TOTAL_NUM_QUEUES; i++)
1072                 dev_idx_to_host_tbl[ena_dev->io_sq_queues[i].idx] = i;
1073
1074         for (i = 0; i < 1 << rss->tbl_log_size; i++) {
1075                 if (rss->rss_ind_tbl[i].cq_idx > ENA_TOTAL_NUM_QUEUES)
1076                         return -EINVAL;
1077                 idx = (u8)rss->rss_ind_tbl[i].cq_idx;
1078
1079                 if (dev_idx_to_host_tbl[idx] > ENA_TOTAL_NUM_QUEUES)
1080                         return -EINVAL;
1081
1082                 rss->host_rss_ind_tbl[i] = dev_idx_to_host_tbl[idx];
1083         }
1084
1085         return 0;
1086 }
1087
1088 static int ena_com_init_interrupt_moderation_table(struct ena_com_dev *ena_dev)
1089 {
1090         size_t size;
1091
1092         size = sizeof(struct ena_intr_moder_entry) * ENA_INTR_MAX_NUM_OF_LEVELS;
1093
1094         ena_dev->intr_moder_tbl =
1095                 devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
1096         if (!ena_dev->intr_moder_tbl)
1097                 return -ENOMEM;
1098
1099         ena_com_config_default_interrupt_moderation_table(ena_dev);
1100
1101         return 0;
1102 }
1103
1104 static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev,
1105                                                  u16 intr_delay_resolution)
1106 {
1107         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
1108         unsigned int i;
1109
1110         if (!intr_delay_resolution) {
1111                 pr_err("Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n");
1112                 intr_delay_resolution = 1;
1113         }
1114         ena_dev->intr_delay_resolution = intr_delay_resolution;
1115
1116         /* update Rx */
1117         for (i = 0; i < ENA_INTR_MAX_NUM_OF_LEVELS; i++)
1118                 intr_moder_tbl[i].intr_moder_interval /= intr_delay_resolution;
1119
1120         /* update Tx */
1121         ena_dev->intr_moder_tx_interval /= intr_delay_resolution;
1122 }
1123
1124 /*****************************************************************************/
1125 /*******************************      API       ******************************/
1126 /*****************************************************************************/
1127
1128 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,
1129                                   struct ena_admin_aq_entry *cmd,
1130                                   size_t cmd_size,
1131                                   struct ena_admin_acq_entry *comp,
1132                                   size_t comp_size)
1133 {
1134         struct ena_comp_ctx *comp_ctx;
1135         int ret;
1136
1137         comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,
1138                                             comp, comp_size);
1139         if (IS_ERR(comp_ctx)) {
1140                 if (comp_ctx == ERR_PTR(-ENODEV))
1141                         pr_debug("Failed to submit command [%ld]\n",
1142                                  PTR_ERR(comp_ctx));
1143                 else
1144                         pr_err("Failed to submit command [%ld]\n",
1145                                PTR_ERR(comp_ctx));
1146
1147                 return PTR_ERR(comp_ctx);
1148         }
1149
1150         ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);
1151         if (unlikely(ret)) {
1152                 if (admin_queue->running_state)
1153                         pr_err("Failed to process command. ret = %d\n", ret);
1154                 else
1155                         pr_debug("Failed to process command. ret = %d\n", ret);
1156         }
1157         return ret;
1158 }
1159
1160 int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
1161                          struct ena_com_io_cq *io_cq)
1162 {
1163         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1164         struct ena_admin_aq_create_cq_cmd create_cmd;
1165         struct ena_admin_acq_create_cq_resp_desc cmd_completion;
1166         int ret;
1167
1168         memset(&create_cmd, 0x0, sizeof(create_cmd));
1169
1170         create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_CQ;
1171
1172         create_cmd.cq_caps_2 |= (io_cq->cdesc_entry_size_in_bytes / 4) &
1173                 ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;
1174         create_cmd.cq_caps_1 |=
1175                 ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK;
1176
1177         create_cmd.msix_vector = io_cq->msix_vector;
1178         create_cmd.cq_depth = io_cq->q_depth;
1179
1180         ret = ena_com_mem_addr_set(ena_dev,
1181                                    &create_cmd.cq_ba,
1182                                    io_cq->cdesc_addr.phys_addr);
1183         if (unlikely(ret)) {
1184                 pr_err("memory address set failed\n");
1185                 return ret;
1186         }
1187
1188         ret = ena_com_execute_admin_command(admin_queue,
1189                                             (struct ena_admin_aq_entry *)&create_cmd,
1190                                             sizeof(create_cmd),
1191                                             (struct ena_admin_acq_entry *)&cmd_completion,
1192                                             sizeof(cmd_completion));
1193         if (unlikely(ret)) {
1194                 pr_err("Failed to create IO CQ. error: %d\n", ret);
1195                 return ret;
1196         }
1197
1198         io_cq->idx = cmd_completion.cq_idx;
1199
1200         io_cq->unmask_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1201                 cmd_completion.cq_interrupt_unmask_register_offset);
1202
1203         if (cmd_completion.cq_head_db_register_offset)
1204                 io_cq->cq_head_db_reg =
1205                         (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1206                         cmd_completion.cq_head_db_register_offset);
1207
1208         if (cmd_completion.numa_node_register_offset)
1209                 io_cq->numa_node_cfg_reg =
1210                         (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1211                         cmd_completion.numa_node_register_offset);
1212
1213         pr_debug("created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth);
1214
1215         return ret;
1216 }
1217
1218 int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid,
1219                             struct ena_com_io_sq **io_sq,
1220                             struct ena_com_io_cq **io_cq)
1221 {
1222         if (qid >= ENA_TOTAL_NUM_QUEUES) {
1223                 pr_err("Invalid queue number %d but the max is %d\n", qid,
1224                        ENA_TOTAL_NUM_QUEUES);
1225                 return -EINVAL;
1226         }
1227
1228         *io_sq = &ena_dev->io_sq_queues[qid];
1229         *io_cq = &ena_dev->io_cq_queues[qid];
1230
1231         return 0;
1232 }
1233
1234 void ena_com_abort_admin_commands(struct ena_com_dev *ena_dev)
1235 {
1236         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1237         struct ena_comp_ctx *comp_ctx;
1238         u16 i;
1239
1240         if (!admin_queue->comp_ctx)
1241                 return;
1242
1243         for (i = 0; i < admin_queue->q_depth; i++) {
1244                 comp_ctx = get_comp_ctxt(admin_queue, i, false);
1245                 if (unlikely(!comp_ctx))
1246                         break;
1247
1248                 comp_ctx->status = ENA_CMD_ABORTED;
1249
1250                 complete(&comp_ctx->wait_event);
1251         }
1252 }
1253
1254 void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev)
1255 {
1256         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1257         unsigned long flags;
1258
1259         spin_lock_irqsave(&admin_queue->q_lock, flags);
1260         while (atomic_read(&admin_queue->outstanding_cmds) != 0) {
1261                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1262                 msleep(ENA_POLL_MS);
1263                 spin_lock_irqsave(&admin_queue->q_lock, flags);
1264         }
1265         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1266 }
1267
1268 int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev,
1269                           struct ena_com_io_cq *io_cq)
1270 {
1271         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1272         struct ena_admin_aq_destroy_cq_cmd destroy_cmd;
1273         struct ena_admin_acq_destroy_cq_resp_desc destroy_resp;
1274         int ret;
1275
1276         memset(&destroy_cmd, 0x0, sizeof(destroy_cmd));
1277
1278         destroy_cmd.cq_idx = io_cq->idx;
1279         destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_CQ;
1280
1281         ret = ena_com_execute_admin_command(admin_queue,
1282                                             (struct ena_admin_aq_entry *)&destroy_cmd,
1283                                             sizeof(destroy_cmd),
1284                                             (struct ena_admin_acq_entry *)&destroy_resp,
1285                                             sizeof(destroy_resp));
1286
1287         if (unlikely(ret && (ret != -ENODEV)))
1288                 pr_err("Failed to destroy IO CQ. error: %d\n", ret);
1289
1290         return ret;
1291 }
1292
1293 bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev)
1294 {
1295         return ena_dev->admin_queue.running_state;
1296 }
1297
1298 void ena_com_set_admin_running_state(struct ena_com_dev *ena_dev, bool state)
1299 {
1300         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1301         unsigned long flags;
1302
1303         spin_lock_irqsave(&admin_queue->q_lock, flags);
1304         ena_dev->admin_queue.running_state = state;
1305         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1306 }
1307
1308 void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev)
1309 {
1310         u16 depth = ena_dev->aenq.q_depth;
1311
1312         WARN(ena_dev->aenq.head != depth, "Invalid AENQ state\n");
1313
1314         /* Init head_db to mark that all entries in the queue
1315          * are initially available
1316          */
1317         writel(depth, ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
1318 }
1319
1320 int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
1321 {
1322         struct ena_com_admin_queue *admin_queue;
1323         struct ena_admin_set_feat_cmd cmd;
1324         struct ena_admin_set_feat_resp resp;
1325         struct ena_admin_get_feat_resp get_resp;
1326         int ret;
1327
1328         ret = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_AENQ_CONFIG);
1329         if (ret) {
1330                 pr_info("Can't get aenq configuration\n");
1331                 return ret;
1332         }
1333
1334         if ((get_resp.u.aenq.supported_groups & groups_flag) != groups_flag) {
1335                 pr_warn("Trying to set unsupported aenq events. supported flag: %x asked flag: %x\n",
1336                         get_resp.u.aenq.supported_groups, groups_flag);
1337                 return -EOPNOTSUPP;
1338         }
1339
1340         memset(&cmd, 0x0, sizeof(cmd));
1341         admin_queue = &ena_dev->admin_queue;
1342
1343         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
1344         cmd.aq_common_descriptor.flags = 0;
1345         cmd.feat_common.feature_id = ENA_ADMIN_AENQ_CONFIG;
1346         cmd.u.aenq.enabled_groups = groups_flag;
1347
1348         ret = ena_com_execute_admin_command(admin_queue,
1349                                             (struct ena_admin_aq_entry *)&cmd,
1350                                             sizeof(cmd),
1351                                             (struct ena_admin_acq_entry *)&resp,
1352                                             sizeof(resp));
1353
1354         if (unlikely(ret))
1355                 pr_err("Failed to config AENQ ret: %d\n", ret);
1356
1357         return ret;
1358 }
1359
1360 int ena_com_get_dma_width(struct ena_com_dev *ena_dev)
1361 {
1362         u32 caps = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
1363         int width;
1364
1365         if (unlikely(caps == ENA_MMIO_READ_TIMEOUT)) {
1366                 pr_err("Reg read timeout occurred\n");
1367                 return -ETIME;
1368         }
1369
1370         width = (caps & ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) >>
1371                 ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT;
1372
1373         pr_debug("ENA dma width: %d\n", width);
1374
1375         if ((width < 32) || width > ENA_MAX_PHYS_ADDR_SIZE_BITS) {
1376                 pr_err("DMA width illegal value: %d\n", width);
1377                 return -EINVAL;
1378         }
1379
1380         ena_dev->dma_addr_bits = width;
1381
1382         return width;
1383 }
1384
1385 int ena_com_validate_version(struct ena_com_dev *ena_dev)
1386 {
1387         u32 ver;
1388         u32 ctrl_ver;
1389         u32 ctrl_ver_masked;
1390
1391         /* Make sure the ENA version and the controller version are at least
1392          * as the driver expects
1393          */
1394         ver = ena_com_reg_bar_read32(ena_dev, ENA_REGS_VERSION_OFF);
1395         ctrl_ver = ena_com_reg_bar_read32(ena_dev,
1396                                           ENA_REGS_CONTROLLER_VERSION_OFF);
1397
1398         if (unlikely((ver == ENA_MMIO_READ_TIMEOUT) ||
1399                      (ctrl_ver == ENA_MMIO_READ_TIMEOUT))) {
1400                 pr_err("Reg read timeout occurred\n");
1401                 return -ETIME;
1402         }
1403
1404         pr_info("ena device version: %d.%d\n",
1405                 (ver & ENA_REGS_VERSION_MAJOR_VERSION_MASK) >>
1406                         ENA_REGS_VERSION_MAJOR_VERSION_SHIFT,
1407                 ver & ENA_REGS_VERSION_MINOR_VERSION_MASK);
1408
1409         if (ver < MIN_ENA_VER) {
1410                 pr_err("ENA version is lower than the minimal version the driver supports\n");
1411                 return -1;
1412         }
1413
1414         pr_info("ena controller version: %d.%d.%d implementation version %d\n",
1415                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) >>
1416                         ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT,
1417                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) >>
1418                         ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT,
1419                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK),
1420                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK) >>
1421                         ENA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT);
1422
1423         ctrl_ver_masked =
1424                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) |
1425                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) |
1426                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK);
1427
1428         /* Validate the ctrl version without the implementation ID */
1429         if (ctrl_ver_masked < MIN_ENA_CTRL_VER) {
1430                 pr_err("ENA ctrl version is lower than the minimal ctrl version the driver supports\n");
1431                 return -1;
1432         }
1433
1434         return 0;
1435 }
1436
1437 void ena_com_admin_destroy(struct ena_com_dev *ena_dev)
1438 {
1439         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1440         struct ena_com_admin_cq *cq = &admin_queue->cq;
1441         struct ena_com_admin_sq *sq = &admin_queue->sq;
1442         struct ena_com_aenq *aenq = &ena_dev->aenq;
1443         u16 size;
1444
1445         if (admin_queue->comp_ctx)
1446                 devm_kfree(ena_dev->dmadev, admin_queue->comp_ctx);
1447         admin_queue->comp_ctx = NULL;
1448         size = ADMIN_SQ_SIZE(admin_queue->q_depth);
1449         if (sq->entries)
1450                 dma_free_coherent(ena_dev->dmadev, size, sq->entries,
1451                                   sq->dma_addr);
1452         sq->entries = NULL;
1453
1454         size = ADMIN_CQ_SIZE(admin_queue->q_depth);
1455         if (cq->entries)
1456                 dma_free_coherent(ena_dev->dmadev, size, cq->entries,
1457                                   cq->dma_addr);
1458         cq->entries = NULL;
1459
1460         size = ADMIN_AENQ_SIZE(aenq->q_depth);
1461         if (ena_dev->aenq.entries)
1462                 dma_free_coherent(ena_dev->dmadev, size, aenq->entries,
1463                                   aenq->dma_addr);
1464         aenq->entries = NULL;
1465 }
1466
1467 void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling)
1468 {
1469         u32 mask_value = 0;
1470
1471         if (polling)
1472                 mask_value = ENA_REGS_ADMIN_INTR_MASK;
1473
1474         writel(mask_value, ena_dev->reg_bar + ENA_REGS_INTR_MASK_OFF);
1475         ena_dev->admin_queue.polling = polling;
1476 }
1477
1478 int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev)
1479 {
1480         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1481
1482         spin_lock_init(&mmio_read->lock);
1483         mmio_read->read_resp =
1484                 dma_zalloc_coherent(ena_dev->dmadev,
1485                                     sizeof(*mmio_read->read_resp),
1486                                     &mmio_read->read_resp_dma_addr, GFP_KERNEL);
1487         if (unlikely(!mmio_read->read_resp))
1488                 return -ENOMEM;
1489
1490         ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
1491
1492         mmio_read->read_resp->req_id = 0x0;
1493         mmio_read->seq_num = 0x0;
1494         mmio_read->readless_supported = true;
1495
1496         return 0;
1497 }
1498
1499 void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported)
1500 {
1501         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1502
1503         mmio_read->readless_supported = readless_supported;
1504 }
1505
1506 void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev)
1507 {
1508         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1509
1510         writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
1511         writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
1512
1513         dma_free_coherent(ena_dev->dmadev, sizeof(*mmio_read->read_resp),
1514                           mmio_read->read_resp, mmio_read->read_resp_dma_addr);
1515
1516         mmio_read->read_resp = NULL;
1517 }
1518
1519 void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev)
1520 {
1521         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1522         u32 addr_low, addr_high;
1523
1524         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(mmio_read->read_resp_dma_addr);
1525         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(mmio_read->read_resp_dma_addr);
1526
1527         writel(addr_low, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
1528         writel(addr_high, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
1529 }
1530
1531 int ena_com_admin_init(struct ena_com_dev *ena_dev,
1532                        struct ena_aenq_handlers *aenq_handlers,
1533                        bool init_spinlock)
1534 {
1535         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1536         u32 aq_caps, acq_caps, dev_sts, addr_low, addr_high;
1537         int ret;
1538
1539         dev_sts = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
1540
1541         if (unlikely(dev_sts == ENA_MMIO_READ_TIMEOUT)) {
1542                 pr_err("Reg read timeout occurred\n");
1543                 return -ETIME;
1544         }
1545
1546         if (!(dev_sts & ENA_REGS_DEV_STS_READY_MASK)) {
1547                 pr_err("Device isn't ready, abort com init\n");
1548                 return -ENODEV;
1549         }
1550
1551         admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;
1552
1553         admin_queue->q_dmadev = ena_dev->dmadev;
1554         admin_queue->polling = false;
1555         admin_queue->curr_cmd_id = 0;
1556
1557         atomic_set(&admin_queue->outstanding_cmds, 0);
1558
1559         if (init_spinlock)
1560                 spin_lock_init(&admin_queue->q_lock);
1561
1562         ret = ena_com_init_comp_ctxt(admin_queue);
1563         if (ret)
1564                 goto error;
1565
1566         ret = ena_com_admin_init_sq(admin_queue);
1567         if (ret)
1568                 goto error;
1569
1570         ret = ena_com_admin_init_cq(admin_queue);
1571         if (ret)
1572                 goto error;
1573
1574         admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1575                 ENA_REGS_AQ_DB_OFF);
1576
1577         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);
1578         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);
1579
1580         writel(addr_low, ena_dev->reg_bar + ENA_REGS_AQ_BASE_LO_OFF);
1581         writel(addr_high, ena_dev->reg_bar + ENA_REGS_AQ_BASE_HI_OFF);
1582
1583         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);
1584         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);
1585
1586         writel(addr_low, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_LO_OFF);
1587         writel(addr_high, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_HI_OFF);
1588
1589         aq_caps = 0;
1590         aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;
1591         aq_caps |= (sizeof(struct ena_admin_aq_entry) <<
1592                         ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT) &
1593                         ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK;
1594
1595         acq_caps = 0;
1596         acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;
1597         acq_caps |= (sizeof(struct ena_admin_acq_entry) <<
1598                 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) &
1599                 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK;
1600
1601         writel(aq_caps, ena_dev->reg_bar + ENA_REGS_AQ_CAPS_OFF);
1602         writel(acq_caps, ena_dev->reg_bar + ENA_REGS_ACQ_CAPS_OFF);
1603         ret = ena_com_admin_init_aenq(ena_dev, aenq_handlers);
1604         if (ret)
1605                 goto error;
1606
1607         admin_queue->running_state = true;
1608
1609         return 0;
1610 error:
1611         ena_com_admin_destroy(ena_dev);
1612
1613         return ret;
1614 }
1615
1616 int ena_com_create_io_queue(struct ena_com_dev *ena_dev,
1617                             struct ena_com_create_io_ctx *ctx)
1618 {
1619         struct ena_com_io_sq *io_sq;
1620         struct ena_com_io_cq *io_cq;
1621         int ret;
1622
1623         if (ctx->qid >= ENA_TOTAL_NUM_QUEUES) {
1624                 pr_err("Qid (%d) is bigger than max num of queues (%d)\n",
1625                        ctx->qid, ENA_TOTAL_NUM_QUEUES);
1626                 return -EINVAL;
1627         }
1628
1629         io_sq = &ena_dev->io_sq_queues[ctx->qid];
1630         io_cq = &ena_dev->io_cq_queues[ctx->qid];
1631
1632         memset(io_sq, 0x0, sizeof(*io_sq));
1633         memset(io_cq, 0x0, sizeof(*io_cq));
1634
1635         /* Init CQ */
1636         io_cq->q_depth = ctx->queue_size;
1637         io_cq->direction = ctx->direction;
1638         io_cq->qid = ctx->qid;
1639
1640         io_cq->msix_vector = ctx->msix_vector;
1641
1642         io_sq->q_depth = ctx->queue_size;
1643         io_sq->direction = ctx->direction;
1644         io_sq->qid = ctx->qid;
1645
1646         io_sq->mem_queue_type = ctx->mem_queue_type;
1647
1648         if (ctx->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
1649                 /* header length is limited to 8 bits */
1650                 io_sq->tx_max_header_size =
1651                         min_t(u32, ena_dev->tx_max_header_size, SZ_256);
1652
1653         ret = ena_com_init_io_sq(ena_dev, ctx, io_sq);
1654         if (ret)
1655                 goto error;
1656         ret = ena_com_init_io_cq(ena_dev, ctx, io_cq);
1657         if (ret)
1658                 goto error;
1659
1660         ret = ena_com_create_io_cq(ena_dev, io_cq);
1661         if (ret)
1662                 goto error;
1663
1664         ret = ena_com_create_io_sq(ena_dev, io_sq, io_cq->idx);
1665         if (ret)
1666                 goto destroy_io_cq;
1667
1668         return 0;
1669
1670 destroy_io_cq:
1671         ena_com_destroy_io_cq(ena_dev, io_cq);
1672 error:
1673         ena_com_io_queue_free(ena_dev, io_sq, io_cq);
1674         return ret;
1675 }
1676
1677 void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid)
1678 {
1679         struct ena_com_io_sq *io_sq;
1680         struct ena_com_io_cq *io_cq;
1681
1682         if (qid >= ENA_TOTAL_NUM_QUEUES) {
1683                 pr_err("Qid (%d) is bigger than max num of queues (%d)\n", qid,
1684                        ENA_TOTAL_NUM_QUEUES);
1685                 return;
1686         }
1687
1688         io_sq = &ena_dev->io_sq_queues[qid];
1689         io_cq = &ena_dev->io_cq_queues[qid];
1690
1691         ena_com_destroy_io_sq(ena_dev, io_sq);
1692         ena_com_destroy_io_cq(ena_dev, io_cq);
1693
1694         ena_com_io_queue_free(ena_dev, io_sq, io_cq);
1695 }
1696
1697 int ena_com_get_link_params(struct ena_com_dev *ena_dev,
1698                             struct ena_admin_get_feat_resp *resp)
1699 {
1700         return ena_com_get_feature(ena_dev, resp, ENA_ADMIN_LINK_CONFIG);
1701 }
1702
1703 int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
1704                               struct ena_com_dev_get_features_ctx *get_feat_ctx)
1705 {
1706         struct ena_admin_get_feat_resp get_resp;
1707         int rc;
1708
1709         rc = ena_com_get_feature(ena_dev, &get_resp,
1710                                  ENA_ADMIN_DEVICE_ATTRIBUTES);
1711         if (rc)
1712                 return rc;
1713
1714         memcpy(&get_feat_ctx->dev_attr, &get_resp.u.dev_attr,
1715                sizeof(get_resp.u.dev_attr));
1716         ena_dev->supported_features = get_resp.u.dev_attr.supported_features;
1717
1718         rc = ena_com_get_feature(ena_dev, &get_resp,
1719                                  ENA_ADMIN_MAX_QUEUES_NUM);
1720         if (rc)
1721                 return rc;
1722
1723         memcpy(&get_feat_ctx->max_queues, &get_resp.u.max_queue,
1724                sizeof(get_resp.u.max_queue));
1725         ena_dev->tx_max_header_size = get_resp.u.max_queue.max_header_size;
1726
1727         rc = ena_com_get_feature(ena_dev, &get_resp,
1728                                  ENA_ADMIN_AENQ_CONFIG);
1729         if (rc)
1730                 return rc;
1731
1732         memcpy(&get_feat_ctx->aenq, &get_resp.u.aenq,
1733                sizeof(get_resp.u.aenq));
1734
1735         rc = ena_com_get_feature(ena_dev, &get_resp,
1736                                  ENA_ADMIN_STATELESS_OFFLOAD_CONFIG);
1737         if (rc)
1738                 return rc;
1739
1740         memcpy(&get_feat_ctx->offload, &get_resp.u.offload,
1741                sizeof(get_resp.u.offload));
1742
1743         /* Driver hints isn't mandatory admin command. So in case the
1744          * command isn't supported set driver hints to 0
1745          */
1746         rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_HW_HINTS);
1747
1748         if (!rc)
1749                 memcpy(&get_feat_ctx->hw_hints, &get_resp.u.hw_hints,
1750                        sizeof(get_resp.u.hw_hints));
1751         else if (rc == -EOPNOTSUPP)
1752                 memset(&get_feat_ctx->hw_hints, 0x0,
1753                        sizeof(get_feat_ctx->hw_hints));
1754         else
1755                 return rc;
1756
1757         return 0;
1758 }
1759
1760 void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev)
1761 {
1762         ena_com_handle_admin_completion(&ena_dev->admin_queue);
1763 }
1764
1765 /* ena_handle_specific_aenq_event:
1766  * return the handler that is relevant to the specific event group
1767  */
1768 static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *dev,
1769                                                      u16 group)
1770 {
1771         struct ena_aenq_handlers *aenq_handlers = dev->aenq.aenq_handlers;
1772
1773         if ((group < ENA_MAX_HANDLERS) && aenq_handlers->handlers[group])
1774                 return aenq_handlers->handlers[group];
1775
1776         return aenq_handlers->unimplemented_handler;
1777 }
1778
1779 /* ena_aenq_intr_handler:
1780  * handles the aenq incoming events.
1781  * pop events from the queue and apply the specific handler
1782  */
1783 void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
1784 {
1785         struct ena_admin_aenq_entry *aenq_e;
1786         struct ena_admin_aenq_common_desc *aenq_common;
1787         struct ena_com_aenq *aenq  = &dev->aenq;
1788         ena_aenq_handler handler_cb;
1789         u16 masked_head, processed = 0;
1790         u8 phase;
1791
1792         masked_head = aenq->head & (aenq->q_depth - 1);
1793         phase = aenq->phase;
1794         aenq_e = &aenq->entries[masked_head]; /* Get first entry */
1795         aenq_common = &aenq_e->aenq_common_desc;
1796
1797         /* Go over all the events */
1798         while ((aenq_common->flags & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) ==
1799                phase) {
1800                 pr_debug("AENQ! Group[%x] Syndrom[%x] timestamp: [%llus]\n",
1801                          aenq_common->group, aenq_common->syndrom,
1802                          (u64)aenq_common->timestamp_low +
1803                                  ((u64)aenq_common->timestamp_high << 32));
1804
1805                 /* Handle specific event*/
1806                 handler_cb = ena_com_get_specific_aenq_cb(dev,
1807                                                           aenq_common->group);
1808                 handler_cb(data, aenq_e); /* call the actual event handler*/
1809
1810                 /* Get next event entry */
1811                 masked_head++;
1812                 processed++;
1813
1814                 if (unlikely(masked_head == aenq->q_depth)) {
1815                         masked_head = 0;
1816                         phase = !phase;
1817                 }
1818                 aenq_e = &aenq->entries[masked_head];
1819                 aenq_common = &aenq_e->aenq_common_desc;
1820         }
1821
1822         aenq->head += processed;
1823         aenq->phase = phase;
1824
1825         /* Don't update aenq doorbell if there weren't any processed events */
1826         if (!processed)
1827                 return;
1828
1829         /* write the aenq doorbell after all AENQ descriptors were read */
1830         mb();
1831         writel_relaxed((u32)aenq->head,
1832                        dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
1833         mmiowb();
1834 }
1835
1836 int ena_com_dev_reset(struct ena_com_dev *ena_dev,
1837                       enum ena_regs_reset_reason_types reset_reason)
1838 {
1839         u32 stat, timeout, cap, reset_val;
1840         int rc;
1841
1842         stat = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
1843         cap = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
1844
1845         if (unlikely((stat == ENA_MMIO_READ_TIMEOUT) ||
1846                      (cap == ENA_MMIO_READ_TIMEOUT))) {
1847                 pr_err("Reg read32 timeout occurred\n");
1848                 return -ETIME;
1849         }
1850
1851         if ((stat & ENA_REGS_DEV_STS_READY_MASK) == 0) {
1852                 pr_err("Device isn't ready, can't reset device\n");
1853                 return -EINVAL;
1854         }
1855
1856         timeout = (cap & ENA_REGS_CAPS_RESET_TIMEOUT_MASK) >>
1857                         ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT;
1858         if (timeout == 0) {
1859                 pr_err("Invalid timeout value\n");
1860                 return -EINVAL;
1861         }
1862
1863         /* start reset */
1864         reset_val = ENA_REGS_DEV_CTL_DEV_RESET_MASK;
1865         reset_val |= (reset_reason << ENA_REGS_DEV_CTL_RESET_REASON_SHIFT) &
1866                      ENA_REGS_DEV_CTL_RESET_REASON_MASK;
1867         writel(reset_val, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
1868
1869         /* Write again the MMIO read request address */
1870         ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
1871
1872         rc = wait_for_reset_state(ena_dev, timeout,
1873                                   ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK);
1874         if (rc != 0) {
1875                 pr_err("Reset indication didn't turn on\n");
1876                 return rc;
1877         }
1878
1879         /* reset done */
1880         writel(0, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
1881         rc = wait_for_reset_state(ena_dev, timeout, 0);
1882         if (rc != 0) {
1883                 pr_err("Reset indication didn't turn off\n");
1884                 return rc;
1885         }
1886
1887         timeout = (cap & ENA_REGS_CAPS_ADMIN_CMD_TO_MASK) >>
1888                 ENA_REGS_CAPS_ADMIN_CMD_TO_SHIFT;
1889         if (timeout)
1890                 /* the resolution of timeout reg is 100ms */
1891                 ena_dev->admin_queue.completion_timeout = timeout * 100000;
1892         else
1893                 ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;
1894
1895         return 0;
1896 }
1897
1898 static int ena_get_dev_stats(struct ena_com_dev *ena_dev,
1899                              struct ena_com_stats_ctx *ctx,
1900                              enum ena_admin_get_stats_type type)
1901 {
1902         struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd;
1903         struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp;
1904         struct ena_com_admin_queue *admin_queue;
1905         int ret;
1906
1907         admin_queue = &ena_dev->admin_queue;
1908
1909         get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS;
1910         get_cmd->aq_common_descriptor.flags = 0;
1911         get_cmd->type = type;
1912
1913         ret =  ena_com_execute_admin_command(admin_queue,
1914                                              (struct ena_admin_aq_entry *)get_cmd,
1915                                              sizeof(*get_cmd),
1916                                              (struct ena_admin_acq_entry *)get_resp,
1917                                              sizeof(*get_resp));
1918
1919         if (unlikely(ret))
1920                 pr_err("Failed to get stats. error: %d\n", ret);
1921
1922         return ret;
1923 }
1924
1925 int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev,
1926                                 struct ena_admin_basic_stats *stats)
1927 {
1928         struct ena_com_stats_ctx ctx;
1929         int ret;
1930
1931         memset(&ctx, 0x0, sizeof(ctx));
1932         ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_BASIC);
1933         if (likely(ret == 0))
1934                 memcpy(stats, &ctx.get_resp.basic_stats,
1935                        sizeof(ctx.get_resp.basic_stats));
1936
1937         return ret;
1938 }
1939
1940 int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu)
1941 {
1942         struct ena_com_admin_queue *admin_queue;
1943         struct ena_admin_set_feat_cmd cmd;
1944         struct ena_admin_set_feat_resp resp;
1945         int ret;
1946
1947         if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) {
1948                 pr_debug("Feature %d isn't supported\n", ENA_ADMIN_MTU);
1949                 return -EOPNOTSUPP;
1950         }
1951
1952         memset(&cmd, 0x0, sizeof(cmd));
1953         admin_queue = &ena_dev->admin_queue;
1954
1955         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
1956         cmd.aq_common_descriptor.flags = 0;
1957         cmd.feat_common.feature_id = ENA_ADMIN_MTU;
1958         cmd.u.mtu.mtu = mtu;
1959
1960         ret = ena_com_execute_admin_command(admin_queue,
1961                                             (struct ena_admin_aq_entry *)&cmd,
1962                                             sizeof(cmd),
1963                                             (struct ena_admin_acq_entry *)&resp,
1964                                             sizeof(resp));
1965
1966         if (unlikely(ret))
1967                 pr_err("Failed to set mtu %d. error: %d\n", mtu, ret);
1968
1969         return ret;
1970 }
1971
1972 int ena_com_get_offload_settings(struct ena_com_dev *ena_dev,
1973                                  struct ena_admin_feature_offload_desc *offload)
1974 {
1975         int ret;
1976         struct ena_admin_get_feat_resp resp;
1977
1978         ret = ena_com_get_feature(ena_dev, &resp,
1979                                   ENA_ADMIN_STATELESS_OFFLOAD_CONFIG);
1980         if (unlikely(ret)) {
1981                 pr_err("Failed to get offload capabilities %d\n", ret);
1982                 return ret;
1983         }
1984
1985         memcpy(offload, &resp.u.offload, sizeof(resp.u.offload));
1986
1987         return 0;
1988 }
1989
1990 int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
1991 {
1992         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1993         struct ena_rss *rss = &ena_dev->rss;
1994         struct ena_admin_set_feat_cmd cmd;
1995         struct ena_admin_set_feat_resp resp;
1996         struct ena_admin_get_feat_resp get_resp;
1997         int ret;
1998
1999         if (!ena_com_check_supported_feature_id(ena_dev,
2000                                                 ENA_ADMIN_RSS_HASH_FUNCTION)) {
2001                 pr_debug("Feature %d isn't supported\n",
2002                          ENA_ADMIN_RSS_HASH_FUNCTION);
2003                 return -EOPNOTSUPP;
2004         }
2005
2006         /* Validate hash function is supported */
2007         ret = ena_com_get_feature(ena_dev, &get_resp,
2008                                   ENA_ADMIN_RSS_HASH_FUNCTION);
2009         if (unlikely(ret))
2010                 return ret;
2011
2012         if (get_resp.u.flow_hash_func.supported_func & (1 << rss->hash_func)) {
2013                 pr_err("Func hash %d isn't supported by device, abort\n",
2014                        rss->hash_func);
2015                 return -EOPNOTSUPP;
2016         }
2017
2018         memset(&cmd, 0x0, sizeof(cmd));
2019
2020         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2021         cmd.aq_common_descriptor.flags =
2022                 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2023         cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_FUNCTION;
2024         cmd.u.flow_hash_func.init_val = rss->hash_init_val;
2025         cmd.u.flow_hash_func.selected_func = 1 << rss->hash_func;
2026
2027         ret = ena_com_mem_addr_set(ena_dev,
2028                                    &cmd.control_buffer.address,
2029                                    rss->hash_key_dma_addr);
2030         if (unlikely(ret)) {
2031                 pr_err("memory address set failed\n");
2032                 return ret;
2033         }
2034
2035         cmd.control_buffer.length = sizeof(*rss->hash_key);
2036
2037         ret = ena_com_execute_admin_command(admin_queue,
2038                                             (struct ena_admin_aq_entry *)&cmd,
2039                                             sizeof(cmd),
2040                                             (struct ena_admin_acq_entry *)&resp,
2041                                             sizeof(resp));
2042         if (unlikely(ret)) {
2043                 pr_err("Failed to set hash function %d. error: %d\n",
2044                        rss->hash_func, ret);
2045                 return -EINVAL;
2046         }
2047
2048         return 0;
2049 }
2050
2051 int ena_com_fill_hash_function(struct ena_com_dev *ena_dev,
2052                                enum ena_admin_hash_functions func,
2053                                const u8 *key, u16 key_len, u32 init_val)
2054 {
2055         struct ena_rss *rss = &ena_dev->rss;
2056         struct ena_admin_get_feat_resp get_resp;
2057         struct ena_admin_feature_rss_flow_hash_control *hash_key =
2058                 rss->hash_key;
2059         int rc;
2060
2061         /* Make sure size is a mult of DWs */
2062         if (unlikely(key_len & 0x3))
2063                 return -EINVAL;
2064
2065         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2066                                     ENA_ADMIN_RSS_HASH_FUNCTION,
2067                                     rss->hash_key_dma_addr,
2068                                     sizeof(*rss->hash_key));
2069         if (unlikely(rc))
2070                 return rc;
2071
2072         if (!((1 << func) & get_resp.u.flow_hash_func.supported_func)) {
2073                 pr_err("Flow hash function %d isn't supported\n", func);
2074                 return -EOPNOTSUPP;
2075         }
2076
2077         switch (func) {
2078         case ENA_ADMIN_TOEPLITZ:
2079                 if (key_len > sizeof(hash_key->key)) {
2080                         pr_err("key len (%hu) is bigger than the max supported (%zu)\n",
2081                                key_len, sizeof(hash_key->key));
2082                         return -EINVAL;
2083                 }
2084
2085                 memcpy(hash_key->key, key, key_len);
2086                 rss->hash_init_val = init_val;
2087                 hash_key->keys_num = key_len >> 2;
2088                 break;
2089         case ENA_ADMIN_CRC32:
2090                 rss->hash_init_val = init_val;
2091                 break;
2092         default:
2093                 pr_err("Invalid hash function (%d)\n", func);
2094                 return -EINVAL;
2095         }
2096
2097         rc = ena_com_set_hash_function(ena_dev);
2098
2099         /* Restore the old function */
2100         if (unlikely(rc))
2101                 ena_com_get_hash_function(ena_dev, NULL, NULL);
2102
2103         return rc;
2104 }
2105
2106 int ena_com_get_hash_function(struct ena_com_dev *ena_dev,
2107                               enum ena_admin_hash_functions *func,
2108                               u8 *key)
2109 {
2110         struct ena_rss *rss = &ena_dev->rss;
2111         struct ena_admin_get_feat_resp get_resp;
2112         struct ena_admin_feature_rss_flow_hash_control *hash_key =
2113                 rss->hash_key;
2114         int rc;
2115
2116         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2117                                     ENA_ADMIN_RSS_HASH_FUNCTION,
2118                                     rss->hash_key_dma_addr,
2119                                     sizeof(*rss->hash_key));
2120         if (unlikely(rc))
2121                 return rc;
2122
2123         rss->hash_func = get_resp.u.flow_hash_func.selected_func;
2124         if (func)
2125                 *func = rss->hash_func;
2126
2127         if (key)
2128                 memcpy(key, hash_key->key, (size_t)(hash_key->keys_num) << 2);
2129
2130         return 0;
2131 }
2132
2133 int ena_com_get_hash_ctrl(struct ena_com_dev *ena_dev,
2134                           enum ena_admin_flow_hash_proto proto,
2135                           u16 *fields)
2136 {
2137         struct ena_rss *rss = &ena_dev->rss;
2138         struct ena_admin_get_feat_resp get_resp;
2139         int rc;
2140
2141         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2142                                     ENA_ADMIN_RSS_HASH_INPUT,
2143                                     rss->hash_ctrl_dma_addr,
2144                                     sizeof(*rss->hash_ctrl));
2145         if (unlikely(rc))
2146                 return rc;
2147
2148         if (fields)
2149                 *fields = rss->hash_ctrl->selected_fields[proto].fields;
2150
2151         return 0;
2152 }
2153
2154 int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
2155 {
2156         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2157         struct ena_rss *rss = &ena_dev->rss;
2158         struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
2159         struct ena_admin_set_feat_cmd cmd;
2160         struct ena_admin_set_feat_resp resp;
2161         int ret;
2162
2163         if (!ena_com_check_supported_feature_id(ena_dev,
2164                                                 ENA_ADMIN_RSS_HASH_INPUT)) {
2165                 pr_debug("Feature %d isn't supported\n",
2166                          ENA_ADMIN_RSS_HASH_INPUT);
2167                 return -EOPNOTSUPP;
2168         }
2169
2170         memset(&cmd, 0x0, sizeof(cmd));
2171
2172         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2173         cmd.aq_common_descriptor.flags =
2174                 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2175         cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_INPUT;
2176         cmd.u.flow_hash_input.enabled_input_sort =
2177                 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK |
2178                 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK;
2179
2180         ret = ena_com_mem_addr_set(ena_dev,
2181                                    &cmd.control_buffer.address,
2182                                    rss->hash_ctrl_dma_addr);
2183         if (unlikely(ret)) {
2184                 pr_err("memory address set failed\n");
2185                 return ret;
2186         }
2187         cmd.control_buffer.length = sizeof(*hash_ctrl);
2188
2189         ret = ena_com_execute_admin_command(admin_queue,
2190                                             (struct ena_admin_aq_entry *)&cmd,
2191                                             sizeof(cmd),
2192                                             (struct ena_admin_acq_entry *)&resp,
2193                                             sizeof(resp));
2194         if (unlikely(ret))
2195                 pr_err("Failed to set hash input. error: %d\n", ret);
2196
2197         return ret;
2198 }
2199
2200 int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
2201 {
2202         struct ena_rss *rss = &ena_dev->rss;
2203         struct ena_admin_feature_rss_hash_control *hash_ctrl =
2204                 rss->hash_ctrl;
2205         u16 available_fields = 0;
2206         int rc, i;
2207
2208         /* Get the supported hash input */
2209         rc = ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2210         if (unlikely(rc))
2211                 return rc;
2212
2213         hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP4].fields =
2214                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2215                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2216
2217         hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP4].fields =
2218                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2219                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2220
2221         hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP6].fields =
2222                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2223                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2224
2225         hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP6].fields =
2226                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2227                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2228
2229         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4].fields =
2230                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2231
2232         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP6].fields =
2233                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2234
2235         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
2236                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2237
2238         hash_ctrl->selected_fields[ENA_ADMIN_RSS_NOT_IP].fields =
2239                 ENA_ADMIN_RSS_L2_DA | ENA_ADMIN_RSS_L2_SA;
2240
2241         for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) {
2242                 available_fields = hash_ctrl->selected_fields[i].fields &
2243                                 hash_ctrl->supported_fields[i].fields;
2244                 if (available_fields != hash_ctrl->selected_fields[i].fields) {
2245                         pr_err("hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n",
2246                                i, hash_ctrl->supported_fields[i].fields,
2247                                hash_ctrl->selected_fields[i].fields);
2248                         return -EOPNOTSUPP;
2249                 }
2250         }
2251
2252         rc = ena_com_set_hash_ctrl(ena_dev);
2253
2254         /* In case of failure, restore the old hash ctrl */
2255         if (unlikely(rc))
2256                 ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2257
2258         return rc;
2259 }
2260
2261 int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev,
2262                            enum ena_admin_flow_hash_proto proto,
2263                            u16 hash_fields)
2264 {
2265         struct ena_rss *rss = &ena_dev->rss;
2266         struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
2267         u16 supported_fields;
2268         int rc;
2269
2270         if (proto >= ENA_ADMIN_RSS_PROTO_NUM) {
2271                 pr_err("Invalid proto num (%u)\n", proto);
2272                 return -EINVAL;
2273         }
2274
2275         /* Get the ctrl table */
2276         rc = ena_com_get_hash_ctrl(ena_dev, proto, NULL);
2277         if (unlikely(rc))
2278                 return rc;
2279
2280         /* Make sure all the fields are supported */
2281         supported_fields = hash_ctrl->supported_fields[proto].fields;
2282         if ((hash_fields & supported_fields) != hash_fields) {
2283                 pr_err("proto %d doesn't support the required fields %x. supports only: %x\n",
2284                        proto, hash_fields, supported_fields);
2285         }
2286
2287         hash_ctrl->selected_fields[proto].fields = hash_fields;
2288
2289         rc = ena_com_set_hash_ctrl(ena_dev);
2290
2291         /* In case of failure, restore the old hash ctrl */
2292         if (unlikely(rc))
2293                 ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2294
2295         return 0;
2296 }
2297
2298 int ena_com_indirect_table_fill_entry(struct ena_com_dev *ena_dev,
2299                                       u16 entry_idx, u16 entry_value)
2300 {
2301         struct ena_rss *rss = &ena_dev->rss;
2302
2303         if (unlikely(entry_idx >= (1 << rss->tbl_log_size)))
2304                 return -EINVAL;
2305
2306         if (unlikely((entry_value > ENA_TOTAL_NUM_QUEUES)))
2307                 return -EINVAL;
2308
2309         rss->host_rss_ind_tbl[entry_idx] = entry_value;
2310
2311         return 0;
2312 }
2313
2314 int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
2315 {
2316         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2317         struct ena_rss *rss = &ena_dev->rss;
2318         struct ena_admin_set_feat_cmd cmd;
2319         struct ena_admin_set_feat_resp resp;
2320         int ret;
2321
2322         if (!ena_com_check_supported_feature_id(
2323                     ena_dev, ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG)) {
2324                 pr_debug("Feature %d isn't supported\n",
2325                          ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
2326                 return -EOPNOTSUPP;
2327         }
2328
2329         ret = ena_com_ind_tbl_convert_to_device(ena_dev);
2330         if (ret) {
2331                 pr_err("Failed to convert host indirection table to device table\n");
2332                 return ret;
2333         }
2334
2335         memset(&cmd, 0x0, sizeof(cmd));
2336
2337         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2338         cmd.aq_common_descriptor.flags =
2339                 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2340         cmd.feat_common.feature_id = ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG;
2341         cmd.u.ind_table.size = rss->tbl_log_size;
2342         cmd.u.ind_table.inline_index = 0xFFFFFFFF;
2343
2344         ret = ena_com_mem_addr_set(ena_dev,
2345                                    &cmd.control_buffer.address,
2346                                    rss->rss_ind_tbl_dma_addr);
2347         if (unlikely(ret)) {
2348                 pr_err("memory address set failed\n");
2349                 return ret;
2350         }
2351
2352         cmd.control_buffer.length = (1ULL << rss->tbl_log_size) *
2353                 sizeof(struct ena_admin_rss_ind_table_entry);
2354
2355         ret = ena_com_execute_admin_command(admin_queue,
2356                                             (struct ena_admin_aq_entry *)&cmd,
2357                                             sizeof(cmd),
2358                                             (struct ena_admin_acq_entry *)&resp,
2359                                             sizeof(resp));
2360
2361         if (unlikely(ret))
2362                 pr_err("Failed to set indirect table. error: %d\n", ret);
2363
2364         return ret;
2365 }
2366
2367 int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl)
2368 {
2369         struct ena_rss *rss = &ena_dev->rss;
2370         struct ena_admin_get_feat_resp get_resp;
2371         u32 tbl_size;
2372         int i, rc;
2373
2374         tbl_size = (1ULL << rss->tbl_log_size) *
2375                 sizeof(struct ena_admin_rss_ind_table_entry);
2376
2377         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2378                                     ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG,
2379                                     rss->rss_ind_tbl_dma_addr,
2380                                     tbl_size);
2381         if (unlikely(rc))
2382                 return rc;
2383
2384         if (!ind_tbl)
2385                 return 0;
2386
2387         rc = ena_com_ind_tbl_convert_from_device(ena_dev);
2388         if (unlikely(rc))
2389                 return rc;
2390
2391         for (i = 0; i < (1 << rss->tbl_log_size); i++)
2392                 ind_tbl[i] = rss->host_rss_ind_tbl[i];
2393
2394         return 0;
2395 }
2396
2397 int ena_com_rss_init(struct ena_com_dev *ena_dev, u16 indr_tbl_log_size)
2398 {
2399         int rc;
2400
2401         memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
2402
2403         rc = ena_com_indirect_table_allocate(ena_dev, indr_tbl_log_size);
2404         if (unlikely(rc))
2405                 goto err_indr_tbl;
2406
2407         rc = ena_com_hash_key_allocate(ena_dev);
2408         if (unlikely(rc))
2409                 goto err_hash_key;
2410
2411         rc = ena_com_hash_ctrl_init(ena_dev);
2412         if (unlikely(rc))
2413                 goto err_hash_ctrl;
2414
2415         return 0;
2416
2417 err_hash_ctrl:
2418         ena_com_hash_key_destroy(ena_dev);
2419 err_hash_key:
2420         ena_com_indirect_table_destroy(ena_dev);
2421 err_indr_tbl:
2422
2423         return rc;
2424 }
2425
2426 void ena_com_rss_destroy(struct ena_com_dev *ena_dev)
2427 {
2428         ena_com_indirect_table_destroy(ena_dev);
2429         ena_com_hash_key_destroy(ena_dev);
2430         ena_com_hash_ctrl_destroy(ena_dev);
2431
2432         memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
2433 }
2434
2435 int ena_com_allocate_host_info(struct ena_com_dev *ena_dev)
2436 {
2437         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2438
2439         host_attr->host_info =
2440                 dma_zalloc_coherent(ena_dev->dmadev, SZ_4K,
2441                                     &host_attr->host_info_dma_addr, GFP_KERNEL);
2442         if (unlikely(!host_attr->host_info))
2443                 return -ENOMEM;
2444
2445         return 0;
2446 }
2447
2448 int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev,
2449                                 u32 debug_area_size)
2450 {
2451         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2452
2453         host_attr->debug_area_virt_addr =
2454                 dma_zalloc_coherent(ena_dev->dmadev, debug_area_size,
2455                                     &host_attr->debug_area_dma_addr, GFP_KERNEL);
2456         if (unlikely(!host_attr->debug_area_virt_addr)) {
2457                 host_attr->debug_area_size = 0;
2458                 return -ENOMEM;
2459         }
2460
2461         host_attr->debug_area_size = debug_area_size;
2462
2463         return 0;
2464 }
2465
2466 void ena_com_delete_host_info(struct ena_com_dev *ena_dev)
2467 {
2468         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2469
2470         if (host_attr->host_info) {
2471                 dma_free_coherent(ena_dev->dmadev, SZ_4K, host_attr->host_info,
2472                                   host_attr->host_info_dma_addr);
2473                 host_attr->host_info = NULL;
2474         }
2475 }
2476
2477 void ena_com_delete_debug_area(struct ena_com_dev *ena_dev)
2478 {
2479         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2480
2481         if (host_attr->debug_area_virt_addr) {
2482                 dma_free_coherent(ena_dev->dmadev, host_attr->debug_area_size,
2483                                   host_attr->debug_area_virt_addr,
2484                                   host_attr->debug_area_dma_addr);
2485                 host_attr->debug_area_virt_addr = NULL;
2486         }
2487 }
2488
2489 int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
2490 {
2491         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2492         struct ena_com_admin_queue *admin_queue;
2493         struct ena_admin_set_feat_cmd cmd;
2494         struct ena_admin_set_feat_resp resp;
2495
2496         int ret;
2497
2498         /* Host attribute config is called before ena_com_get_dev_attr_feat
2499          * so ena_com can't check if the feature is supported.
2500          */
2501
2502         memset(&cmd, 0x0, sizeof(cmd));
2503         admin_queue = &ena_dev->admin_queue;
2504
2505         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2506         cmd.feat_common.feature_id = ENA_ADMIN_HOST_ATTR_CONFIG;
2507
2508         ret = ena_com_mem_addr_set(ena_dev,
2509                                    &cmd.u.host_attr.debug_ba,
2510                                    host_attr->debug_area_dma_addr);
2511         if (unlikely(ret)) {
2512                 pr_err("memory address set failed\n");
2513                 return ret;
2514         }
2515
2516         ret = ena_com_mem_addr_set(ena_dev,
2517                                    &cmd.u.host_attr.os_info_ba,
2518                                    host_attr->host_info_dma_addr);
2519         if (unlikely(ret)) {
2520                 pr_err("memory address set failed\n");
2521                 return ret;
2522         }
2523
2524         cmd.u.host_attr.debug_area_size = host_attr->debug_area_size;
2525
2526         ret = ena_com_execute_admin_command(admin_queue,
2527                                             (struct ena_admin_aq_entry *)&cmd,
2528                                             sizeof(cmd),
2529                                             (struct ena_admin_acq_entry *)&resp,
2530                                             sizeof(resp));
2531
2532         if (unlikely(ret))
2533                 pr_err("Failed to set host attributes: %d\n", ret);
2534
2535         return ret;
2536 }
2537
2538 /* Interrupt moderation */
2539 bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev)
2540 {
2541         return ena_com_check_supported_feature_id(ena_dev,
2542                                                   ENA_ADMIN_INTERRUPT_MODERATION);
2543 }
2544
2545 int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
2546                                                       u32 tx_coalesce_usecs)
2547 {
2548         if (!ena_dev->intr_delay_resolution) {
2549                 pr_err("Illegal interrupt delay granularity value\n");
2550                 return -EFAULT;
2551         }
2552
2553         ena_dev->intr_moder_tx_interval = tx_coalesce_usecs /
2554                 ena_dev->intr_delay_resolution;
2555
2556         return 0;
2557 }
2558
2559 int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
2560                                                       u32 rx_coalesce_usecs)
2561 {
2562         if (!ena_dev->intr_delay_resolution) {
2563                 pr_err("Illegal interrupt delay granularity value\n");
2564                 return -EFAULT;
2565         }
2566
2567         /* We use LOWEST entry of moderation table for storing
2568          * nonadaptive interrupt coalescing values
2569          */
2570         ena_dev->intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
2571                 rx_coalesce_usecs / ena_dev->intr_delay_resolution;
2572
2573         return 0;
2574 }
2575
2576 void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev)
2577 {
2578         if (ena_dev->intr_moder_tbl)
2579                 devm_kfree(ena_dev->dmadev, ena_dev->intr_moder_tbl);
2580         ena_dev->intr_moder_tbl = NULL;
2581 }
2582
2583 int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev)
2584 {
2585         struct ena_admin_get_feat_resp get_resp;
2586         u16 delay_resolution;
2587         int rc;
2588
2589         rc = ena_com_get_feature(ena_dev, &get_resp,
2590                                  ENA_ADMIN_INTERRUPT_MODERATION);
2591
2592         if (rc) {
2593                 if (rc == -EOPNOTSUPP) {
2594                         pr_debug("Feature %d isn't supported\n",
2595                                  ENA_ADMIN_INTERRUPT_MODERATION);
2596                         rc = 0;
2597                 } else {
2598                         pr_err("Failed to get interrupt moderation admin cmd. rc: %d\n",
2599                                rc);
2600                 }
2601
2602                 /* no moderation supported, disable adaptive support */
2603                 ena_com_disable_adaptive_moderation(ena_dev);
2604                 return rc;
2605         }
2606
2607         rc = ena_com_init_interrupt_moderation_table(ena_dev);
2608         if (rc)
2609                 goto err;
2610
2611         /* if moderation is supported by device we set adaptive moderation */
2612         delay_resolution = get_resp.u.intr_moderation.intr_delay_resolution;
2613         ena_com_update_intr_delay_resolution(ena_dev, delay_resolution);
2614         ena_com_enable_adaptive_moderation(ena_dev);
2615
2616         return 0;
2617 err:
2618         ena_com_destroy_interrupt_moderation(ena_dev);
2619         return rc;
2620 }
2621
2622 void ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev)
2623 {
2624         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2625
2626         if (!intr_moder_tbl)
2627                 return;
2628
2629         intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
2630                 ENA_INTR_LOWEST_USECS;
2631         intr_moder_tbl[ENA_INTR_MODER_LOWEST].pkts_per_interval =
2632                 ENA_INTR_LOWEST_PKTS;
2633         intr_moder_tbl[ENA_INTR_MODER_LOWEST].bytes_per_interval =
2634                 ENA_INTR_LOWEST_BYTES;
2635
2636         intr_moder_tbl[ENA_INTR_MODER_LOW].intr_moder_interval =
2637                 ENA_INTR_LOW_USECS;
2638         intr_moder_tbl[ENA_INTR_MODER_LOW].pkts_per_interval =
2639                 ENA_INTR_LOW_PKTS;
2640         intr_moder_tbl[ENA_INTR_MODER_LOW].bytes_per_interval =
2641                 ENA_INTR_LOW_BYTES;
2642
2643         intr_moder_tbl[ENA_INTR_MODER_MID].intr_moder_interval =
2644                 ENA_INTR_MID_USECS;
2645         intr_moder_tbl[ENA_INTR_MODER_MID].pkts_per_interval =
2646                 ENA_INTR_MID_PKTS;
2647         intr_moder_tbl[ENA_INTR_MODER_MID].bytes_per_interval =
2648                 ENA_INTR_MID_BYTES;
2649
2650         intr_moder_tbl[ENA_INTR_MODER_HIGH].intr_moder_interval =
2651                 ENA_INTR_HIGH_USECS;
2652         intr_moder_tbl[ENA_INTR_MODER_HIGH].pkts_per_interval =
2653                 ENA_INTR_HIGH_PKTS;
2654         intr_moder_tbl[ENA_INTR_MODER_HIGH].bytes_per_interval =
2655                 ENA_INTR_HIGH_BYTES;
2656
2657         intr_moder_tbl[ENA_INTR_MODER_HIGHEST].intr_moder_interval =
2658                 ENA_INTR_HIGHEST_USECS;
2659         intr_moder_tbl[ENA_INTR_MODER_HIGHEST].pkts_per_interval =
2660                 ENA_INTR_HIGHEST_PKTS;
2661         intr_moder_tbl[ENA_INTR_MODER_HIGHEST].bytes_per_interval =
2662                 ENA_INTR_HIGHEST_BYTES;
2663 }
2664
2665 unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev)
2666 {
2667         return ena_dev->intr_moder_tx_interval;
2668 }
2669
2670 unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev)
2671 {
2672         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2673
2674         if (intr_moder_tbl)
2675                 return intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval;
2676
2677         return 0;
2678 }
2679
2680 void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev,
2681                                         enum ena_intr_moder_level level,
2682                                         struct ena_intr_moder_entry *entry)
2683 {
2684         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2685
2686         if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
2687                 return;
2688
2689         intr_moder_tbl[level].intr_moder_interval = entry->intr_moder_interval;
2690         if (ena_dev->intr_delay_resolution)
2691                 intr_moder_tbl[level].intr_moder_interval /=
2692                         ena_dev->intr_delay_resolution;
2693         intr_moder_tbl[level].pkts_per_interval = entry->pkts_per_interval;
2694
2695         /* use hardcoded value until ethtool supports bytecount parameter */
2696         if (entry->bytes_per_interval != ENA_INTR_BYTE_COUNT_NOT_SUPPORTED)
2697                 intr_moder_tbl[level].bytes_per_interval = entry->bytes_per_interval;
2698 }
2699
2700 void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
2701                                        enum ena_intr_moder_level level,
2702                                        struct ena_intr_moder_entry *entry)
2703 {
2704         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2705
2706         if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
2707                 return;
2708
2709         entry->intr_moder_interval = intr_moder_tbl[level].intr_moder_interval;
2710         if (ena_dev->intr_delay_resolution)
2711                 entry->intr_moder_interval *= ena_dev->intr_delay_resolution;
2712         entry->pkts_per_interval =
2713         intr_moder_tbl[level].pkts_per_interval;
2714         entry->bytes_per_interval = intr_moder_tbl[level].bytes_per_interval;
2715 }