1b9d3130af4d64f844498d19b93adf8da0f2ebc6
[sfrench/cifs-2.6.git] / drivers / net / ethernet / amazon / ena / ena_com.c
1 /*
2  * Copyright 2015 Amazon.com, Inc. or its affiliates.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32
33 #include "ena_com.h"
34
35 /*****************************************************************************/
36 /*****************************************************************************/
37
38 /* Timeout in micro-sec */
39 #define ADMIN_CMD_TIMEOUT_US (3000000)
40
41 #define ENA_ASYNC_QUEUE_DEPTH 16
42 #define ENA_ADMIN_QUEUE_DEPTH 32
43
44 #define MIN_ENA_VER (((ENA_COMMON_SPEC_VERSION_MAJOR) << \
45                 ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) \
46                 | (ENA_COMMON_SPEC_VERSION_MINOR))
47
48 #define ENA_CTRL_MAJOR          0
49 #define ENA_CTRL_MINOR          0
50 #define ENA_CTRL_SUB_MINOR      1
51
52 #define MIN_ENA_CTRL_VER \
53         (((ENA_CTRL_MAJOR) << \
54         (ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT)) | \
55         ((ENA_CTRL_MINOR) << \
56         (ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT)) | \
57         (ENA_CTRL_SUB_MINOR))
58
59 #define ENA_DMA_ADDR_TO_UINT32_LOW(x)   ((u32)((u64)(x)))
60 #define ENA_DMA_ADDR_TO_UINT32_HIGH(x)  ((u32)(((u64)(x)) >> 32))
61
62 #define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF
63
64 #define ENA_REGS_ADMIN_INTR_MASK 1
65
66 #define ENA_POLL_MS     5
67
68 /*****************************************************************************/
69 /*****************************************************************************/
70 /*****************************************************************************/
71
72 enum ena_cmd_status {
73         ENA_CMD_SUBMITTED,
74         ENA_CMD_COMPLETED,
75         /* Abort - canceled by the driver */
76         ENA_CMD_ABORTED,
77 };
78
79 struct ena_comp_ctx {
80         struct completion wait_event;
81         struct ena_admin_acq_entry *user_cqe;
82         u32 comp_size;
83         enum ena_cmd_status status;
84         /* status from the device */
85         u8 comp_status;
86         u8 cmd_opcode;
87         bool occupied;
88 };
89
90 struct ena_com_stats_ctx {
91         struct ena_admin_aq_get_stats_cmd get_cmd;
92         struct ena_admin_acq_get_stats_resp get_resp;
93 };
94
95 static inline int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
96                                        struct ena_common_mem_addr *ena_addr,
97                                        dma_addr_t addr)
98 {
99         if ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr) {
100                 pr_err("dma address has more bits that the device supports\n");
101                 return -EINVAL;
102         }
103
104         ena_addr->mem_addr_low = lower_32_bits(addr);
105         ena_addr->mem_addr_high = (u16)upper_32_bits(addr);
106
107         return 0;
108 }
109
110 static int ena_com_admin_init_sq(struct ena_com_admin_queue *queue)
111 {
112         struct ena_com_admin_sq *sq = &queue->sq;
113         u16 size = ADMIN_SQ_SIZE(queue->q_depth);
114
115         sq->entries = dma_zalloc_coherent(queue->q_dmadev, size, &sq->dma_addr,
116                                           GFP_KERNEL);
117
118         if (!sq->entries) {
119                 pr_err("memory allocation failed");
120                 return -ENOMEM;
121         }
122
123         sq->head = 0;
124         sq->tail = 0;
125         sq->phase = 1;
126
127         sq->db_addr = NULL;
128
129         return 0;
130 }
131
132 static int ena_com_admin_init_cq(struct ena_com_admin_queue *queue)
133 {
134         struct ena_com_admin_cq *cq = &queue->cq;
135         u16 size = ADMIN_CQ_SIZE(queue->q_depth);
136
137         cq->entries = dma_zalloc_coherent(queue->q_dmadev, size, &cq->dma_addr,
138                                           GFP_KERNEL);
139
140         if (!cq->entries) {
141                 pr_err("memory allocation failed");
142                 return -ENOMEM;
143         }
144
145         cq->head = 0;
146         cq->phase = 1;
147
148         return 0;
149 }
150
151 static int ena_com_admin_init_aenq(struct ena_com_dev *dev,
152                                    struct ena_aenq_handlers *aenq_handlers)
153 {
154         struct ena_com_aenq *aenq = &dev->aenq;
155         u32 addr_low, addr_high, aenq_caps;
156         u16 size;
157
158         dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH;
159         size = ADMIN_AENQ_SIZE(ENA_ASYNC_QUEUE_DEPTH);
160         aenq->entries = dma_zalloc_coherent(dev->dmadev, size, &aenq->dma_addr,
161                                             GFP_KERNEL);
162
163         if (!aenq->entries) {
164                 pr_err("memory allocation failed");
165                 return -ENOMEM;
166         }
167
168         aenq->head = aenq->q_depth;
169         aenq->phase = 1;
170
171         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(aenq->dma_addr);
172         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(aenq->dma_addr);
173
174         writel(addr_low, dev->reg_bar + ENA_REGS_AENQ_BASE_LO_OFF);
175         writel(addr_high, dev->reg_bar + ENA_REGS_AENQ_BASE_HI_OFF);
176
177         aenq_caps = 0;
178         aenq_caps |= dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK;
179         aenq_caps |= (sizeof(struct ena_admin_aenq_entry)
180                       << ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) &
181                      ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK;
182         writel(aenq_caps, dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF);
183
184         if (unlikely(!aenq_handlers)) {
185                 pr_err("aenq handlers pointer is NULL\n");
186                 return -EINVAL;
187         }
188
189         aenq->aenq_handlers = aenq_handlers;
190
191         return 0;
192 }
193
194 static inline void comp_ctxt_release(struct ena_com_admin_queue *queue,
195                                      struct ena_comp_ctx *comp_ctx)
196 {
197         comp_ctx->occupied = false;
198         atomic_dec(&queue->outstanding_cmds);
199 }
200
201 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *queue,
202                                           u16 command_id, bool capture)
203 {
204         if (unlikely(command_id >= queue->q_depth)) {
205                 pr_err("command id is larger than the queue size. cmd_id: %u queue size %d\n",
206                        command_id, queue->q_depth);
207                 return NULL;
208         }
209
210         if (unlikely(queue->comp_ctx[command_id].occupied && capture)) {
211                 pr_err("Completion context is occupied\n");
212                 return NULL;
213         }
214
215         if (capture) {
216                 atomic_inc(&queue->outstanding_cmds);
217                 queue->comp_ctx[command_id].occupied = true;
218         }
219
220         return &queue->comp_ctx[command_id];
221 }
222
223 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
224                                                        struct ena_admin_aq_entry *cmd,
225                                                        size_t cmd_size_in_bytes,
226                                                        struct ena_admin_acq_entry *comp,
227                                                        size_t comp_size_in_bytes)
228 {
229         struct ena_comp_ctx *comp_ctx;
230         u16 tail_masked, cmd_id;
231         u16 queue_size_mask;
232         u16 cnt;
233
234         queue_size_mask = admin_queue->q_depth - 1;
235
236         tail_masked = admin_queue->sq.tail & queue_size_mask;
237
238         /* In case of queue FULL */
239         cnt = atomic_read(&admin_queue->outstanding_cmds);
240         if (cnt >= admin_queue->q_depth) {
241                 pr_debug("admin queue is full.\n");
242                 admin_queue->stats.out_of_space++;
243                 return ERR_PTR(-ENOSPC);
244         }
245
246         cmd_id = admin_queue->curr_cmd_id;
247
248         cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &
249                 ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK;
250
251         cmd->aq_common_descriptor.command_id |= cmd_id &
252                 ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
253
254         comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);
255         if (unlikely(!comp_ctx))
256                 return ERR_PTR(-EINVAL);
257
258         comp_ctx->status = ENA_CMD_SUBMITTED;
259         comp_ctx->comp_size = (u32)comp_size_in_bytes;
260         comp_ctx->user_cqe = comp;
261         comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode;
262
263         reinit_completion(&comp_ctx->wait_event);
264
265         memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);
266
267         admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &
268                 queue_size_mask;
269
270         admin_queue->sq.tail++;
271         admin_queue->stats.submitted_cmd++;
272
273         if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))
274                 admin_queue->sq.phase = !admin_queue->sq.phase;
275
276         writel(admin_queue->sq.tail, admin_queue->sq.db_addr);
277
278         return comp_ctx;
279 }
280
281 static inline int ena_com_init_comp_ctxt(struct ena_com_admin_queue *queue)
282 {
283         size_t size = queue->q_depth * sizeof(struct ena_comp_ctx);
284         struct ena_comp_ctx *comp_ctx;
285         u16 i;
286
287         queue->comp_ctx = devm_kzalloc(queue->q_dmadev, size, GFP_KERNEL);
288         if (unlikely(!queue->comp_ctx)) {
289                 pr_err("memory allocation failed");
290                 return -ENOMEM;
291         }
292
293         for (i = 0; i < queue->q_depth; i++) {
294                 comp_ctx = get_comp_ctxt(queue, i, false);
295                 if (comp_ctx)
296                         init_completion(&comp_ctx->wait_event);
297         }
298
299         return 0;
300 }
301
302 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
303                                                      struct ena_admin_aq_entry *cmd,
304                                                      size_t cmd_size_in_bytes,
305                                                      struct ena_admin_acq_entry *comp,
306                                                      size_t comp_size_in_bytes)
307 {
308         unsigned long flags;
309         struct ena_comp_ctx *comp_ctx;
310
311         spin_lock_irqsave(&admin_queue->q_lock, flags);
312         if (unlikely(!admin_queue->running_state)) {
313                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
314                 return ERR_PTR(-ENODEV);
315         }
316         comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,
317                                               cmd_size_in_bytes,
318                                               comp,
319                                               comp_size_in_bytes);
320         if (IS_ERR(comp_ctx))
321                 admin_queue->running_state = false;
322         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
323
324         return comp_ctx;
325 }
326
327 static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
328                               struct ena_com_create_io_ctx *ctx,
329                               struct ena_com_io_sq *io_sq)
330 {
331         size_t size;
332         int dev_node = 0;
333
334         memset(&io_sq->desc_addr, 0x0, sizeof(io_sq->desc_addr));
335
336         io_sq->desc_entry_size =
337                 (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
338                 sizeof(struct ena_eth_io_tx_desc) :
339                 sizeof(struct ena_eth_io_rx_desc);
340
341         size = io_sq->desc_entry_size * io_sq->q_depth;
342
343         if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) {
344                 dev_node = dev_to_node(ena_dev->dmadev);
345                 set_dev_node(ena_dev->dmadev, ctx->numa_node);
346                 io_sq->desc_addr.virt_addr =
347                         dma_zalloc_coherent(ena_dev->dmadev, size,
348                                             &io_sq->desc_addr.phys_addr,
349                                             GFP_KERNEL);
350                 set_dev_node(ena_dev->dmadev, dev_node);
351                 if (!io_sq->desc_addr.virt_addr) {
352                         io_sq->desc_addr.virt_addr =
353                                 dma_zalloc_coherent(ena_dev->dmadev, size,
354                                                     &io_sq->desc_addr.phys_addr,
355                                                     GFP_KERNEL);
356                 }
357         } else {
358                 dev_node = dev_to_node(ena_dev->dmadev);
359                 set_dev_node(ena_dev->dmadev, ctx->numa_node);
360                 io_sq->desc_addr.virt_addr =
361                         devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
362                 set_dev_node(ena_dev->dmadev, dev_node);
363                 if (!io_sq->desc_addr.virt_addr) {
364                         io_sq->desc_addr.virt_addr =
365                                 devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
366                 }
367         }
368
369         if (!io_sq->desc_addr.virt_addr) {
370                 pr_err("memory allocation failed");
371                 return -ENOMEM;
372         }
373
374         io_sq->tail = 0;
375         io_sq->next_to_comp = 0;
376         io_sq->phase = 1;
377
378         return 0;
379 }
380
381 static int ena_com_init_io_cq(struct ena_com_dev *ena_dev,
382                               struct ena_com_create_io_ctx *ctx,
383                               struct ena_com_io_cq *io_cq)
384 {
385         size_t size;
386         int prev_node = 0;
387
388         memset(&io_cq->cdesc_addr, 0x0, sizeof(io_cq->cdesc_addr));
389
390         /* Use the basic completion descriptor for Rx */
391         io_cq->cdesc_entry_size_in_bytes =
392                 (io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
393                 sizeof(struct ena_eth_io_tx_cdesc) :
394                 sizeof(struct ena_eth_io_rx_cdesc_base);
395
396         size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
397
398         prev_node = dev_to_node(ena_dev->dmadev);
399         set_dev_node(ena_dev->dmadev, ctx->numa_node);
400         io_cq->cdesc_addr.virt_addr =
401                 dma_zalloc_coherent(ena_dev->dmadev, size,
402                                     &io_cq->cdesc_addr.phys_addr, GFP_KERNEL);
403         set_dev_node(ena_dev->dmadev, prev_node);
404         if (!io_cq->cdesc_addr.virt_addr) {
405                 io_cq->cdesc_addr.virt_addr =
406                         dma_zalloc_coherent(ena_dev->dmadev, size,
407                                             &io_cq->cdesc_addr.phys_addr,
408                                             GFP_KERNEL);
409         }
410
411         if (!io_cq->cdesc_addr.virt_addr) {
412                 pr_err("memory allocation failed");
413                 return -ENOMEM;
414         }
415
416         io_cq->phase = 1;
417         io_cq->head = 0;
418
419         return 0;
420 }
421
422 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,
423                                                    struct ena_admin_acq_entry *cqe)
424 {
425         struct ena_comp_ctx *comp_ctx;
426         u16 cmd_id;
427
428         cmd_id = cqe->acq_common_descriptor.command &
429                 ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;
430
431         comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);
432         if (unlikely(!comp_ctx)) {
433                 pr_err("comp_ctx is NULL. Changing the admin queue running state\n");
434                 admin_queue->running_state = false;
435                 return;
436         }
437
438         comp_ctx->status = ENA_CMD_COMPLETED;
439         comp_ctx->comp_status = cqe->acq_common_descriptor.status;
440
441         if (comp_ctx->user_cqe)
442                 memcpy(comp_ctx->user_cqe, (void *)cqe, comp_ctx->comp_size);
443
444         if (!admin_queue->polling)
445                 complete(&comp_ctx->wait_event);
446 }
447
448 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)
449 {
450         struct ena_admin_acq_entry *cqe = NULL;
451         u16 comp_num = 0;
452         u16 head_masked;
453         u8 phase;
454
455         head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);
456         phase = admin_queue->cq.phase;
457
458         cqe = &admin_queue->cq.entries[head_masked];
459
460         /* Go over all the completions */
461         while ((cqe->acq_common_descriptor.flags &
462                         ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) {
463                 /* Do not read the rest of the completion entry before the
464                  * phase bit was validated
465                  */
466                 rmb();
467                 ena_com_handle_single_admin_completion(admin_queue, cqe);
468
469                 head_masked++;
470                 comp_num++;
471                 if (unlikely(head_masked == admin_queue->q_depth)) {
472                         head_masked = 0;
473                         phase = !phase;
474                 }
475
476                 cqe = &admin_queue->cq.entries[head_masked];
477         }
478
479         admin_queue->cq.head += comp_num;
480         admin_queue->cq.phase = phase;
481         admin_queue->sq.head += comp_num;
482         admin_queue->stats.completed_cmd += comp_num;
483 }
484
485 static int ena_com_comp_status_to_errno(u8 comp_status)
486 {
487         if (unlikely(comp_status != 0))
488                 pr_err("admin command failed[%u]\n", comp_status);
489
490         if (unlikely(comp_status > ENA_ADMIN_UNKNOWN_ERROR))
491                 return -EINVAL;
492
493         switch (comp_status) {
494         case ENA_ADMIN_SUCCESS:
495                 return 0;
496         case ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE:
497                 return -ENOMEM;
498         case ENA_ADMIN_UNSUPPORTED_OPCODE:
499                 return -EOPNOTSUPP;
500         case ENA_ADMIN_BAD_OPCODE:
501         case ENA_ADMIN_MALFORMED_REQUEST:
502         case ENA_ADMIN_ILLEGAL_PARAMETER:
503         case ENA_ADMIN_UNKNOWN_ERROR:
504                 return -EINVAL;
505         }
506
507         return 0;
508 }
509
510 static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_ctx,
511                                                      struct ena_com_admin_queue *admin_queue)
512 {
513         unsigned long flags, timeout;
514         int ret;
515
516         timeout = jiffies + usecs_to_jiffies(admin_queue->completion_timeout);
517
518         while (1) {
519                 spin_lock_irqsave(&admin_queue->q_lock, flags);
520                 ena_com_handle_admin_completion(admin_queue);
521                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
522
523                 if (comp_ctx->status != ENA_CMD_SUBMITTED)
524                         break;
525
526                 if (time_is_before_jiffies(timeout)) {
527                         pr_err("Wait for completion (polling) timeout\n");
528                         /* ENA didn't have any completion */
529                         spin_lock_irqsave(&admin_queue->q_lock, flags);
530                         admin_queue->stats.no_completion++;
531                         admin_queue->running_state = false;
532                         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
533
534                         ret = -ETIME;
535                         goto err;
536                 }
537
538                 msleep(ENA_POLL_MS);
539         }
540
541         if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) {
542                 pr_err("Command was aborted\n");
543                 spin_lock_irqsave(&admin_queue->q_lock, flags);
544                 admin_queue->stats.aborted_cmd++;
545                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
546                 ret = -ENODEV;
547                 goto err;
548         }
549
550         WARN(comp_ctx->status != ENA_CMD_COMPLETED, "Invalid comp status %d\n",
551              comp_ctx->status);
552
553         ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
554 err:
555         comp_ctxt_release(admin_queue, comp_ctx);
556         return ret;
557 }
558
559 static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx,
560                                                         struct ena_com_admin_queue *admin_queue)
561 {
562         unsigned long flags;
563         int ret;
564
565         wait_for_completion_timeout(&comp_ctx->wait_event,
566                                     usecs_to_jiffies(
567                                             admin_queue->completion_timeout));
568
569         /* In case the command wasn't completed find out the root cause.
570          * There might be 2 kinds of errors
571          * 1) No completion (timeout reached)
572          * 2) There is completion but the device didn't get any msi-x interrupt.
573          */
574         if (unlikely(comp_ctx->status == ENA_CMD_SUBMITTED)) {
575                 spin_lock_irqsave(&admin_queue->q_lock, flags);
576                 ena_com_handle_admin_completion(admin_queue);
577                 admin_queue->stats.no_completion++;
578                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
579
580                 if (comp_ctx->status == ENA_CMD_COMPLETED)
581                         pr_err("The ena device have completion but the driver didn't receive any MSI-X interrupt (cmd %d)\n",
582                                comp_ctx->cmd_opcode);
583                 else
584                         pr_err("The ena device doesn't send any completion for the admin cmd %d status %d\n",
585                                comp_ctx->cmd_opcode, comp_ctx->status);
586
587                 admin_queue->running_state = false;
588                 ret = -ETIME;
589                 goto err;
590         }
591
592         ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
593 err:
594         comp_ctxt_release(admin_queue, comp_ctx);
595         return ret;
596 }
597
598 /* This method read the hardware device register through posting writes
599  * and waiting for response
600  * On timeout the function will return ENA_MMIO_READ_TIMEOUT
601  */
602 static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
603 {
604         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
605         volatile struct ena_admin_ena_mmio_req_read_less_resp *read_resp =
606                 mmio_read->read_resp;
607         u32 mmio_read_reg, ret, i;
608         unsigned long flags;
609         u32 timeout = mmio_read->reg_read_to;
610
611         might_sleep();
612
613         if (timeout == 0)
614                 timeout = ENA_REG_READ_TIMEOUT;
615
616         /* If readless is disabled, perform regular read */
617         if (!mmio_read->readless_supported)
618                 return readl(ena_dev->reg_bar + offset);
619
620         spin_lock_irqsave(&mmio_read->lock, flags);
621         mmio_read->seq_num++;
622
623         read_resp->req_id = mmio_read->seq_num + 0xDEAD;
624         mmio_read_reg = (offset << ENA_REGS_MMIO_REG_READ_REG_OFF_SHIFT) &
625                         ENA_REGS_MMIO_REG_READ_REG_OFF_MASK;
626         mmio_read_reg |= mmio_read->seq_num &
627                         ENA_REGS_MMIO_REG_READ_REQ_ID_MASK;
628
629         /* make sure read_resp->req_id get updated before the hw can write
630          * there
631          */
632         wmb();
633
634         writel_relaxed(mmio_read_reg,
635                        ena_dev->reg_bar + ENA_REGS_MMIO_REG_READ_OFF);
636
637         mmiowb();
638         for (i = 0; i < timeout; i++) {
639                 if (read_resp->req_id == mmio_read->seq_num)
640                         break;
641
642                 udelay(1);
643         }
644
645         if (unlikely(i == timeout)) {
646                 pr_err("reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n",
647                        mmio_read->seq_num, offset, read_resp->req_id,
648                        read_resp->reg_off);
649                 ret = ENA_MMIO_READ_TIMEOUT;
650                 goto err;
651         }
652
653         if (read_resp->reg_off != offset) {
654                 pr_err("Read failure: wrong offset provided");
655                 ret = ENA_MMIO_READ_TIMEOUT;
656         } else {
657                 ret = read_resp->reg_val;
658         }
659 err:
660         spin_unlock_irqrestore(&mmio_read->lock, flags);
661
662         return ret;
663 }
664
665 /* There are two types to wait for completion.
666  * Polling mode - wait until the completion is available.
667  * Async mode - wait on wait queue until the completion is ready
668  * (or the timeout expired).
669  * It is expected that the IRQ called ena_com_handle_admin_completion
670  * to mark the completions.
671  */
672 static int ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx,
673                                              struct ena_com_admin_queue *admin_queue)
674 {
675         if (admin_queue->polling)
676                 return ena_com_wait_and_process_admin_cq_polling(comp_ctx,
677                                                                  admin_queue);
678
679         return ena_com_wait_and_process_admin_cq_interrupts(comp_ctx,
680                                                             admin_queue);
681 }
682
683 static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev,
684                                  struct ena_com_io_sq *io_sq)
685 {
686         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
687         struct ena_admin_aq_destroy_sq_cmd destroy_cmd;
688         struct ena_admin_acq_destroy_sq_resp_desc destroy_resp;
689         u8 direction;
690         int ret;
691
692         memset(&destroy_cmd, 0x0, sizeof(destroy_cmd));
693
694         if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
695                 direction = ENA_ADMIN_SQ_DIRECTION_TX;
696         else
697                 direction = ENA_ADMIN_SQ_DIRECTION_RX;
698
699         destroy_cmd.sq.sq_identity |= (direction <<
700                 ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) &
701                 ENA_ADMIN_SQ_SQ_DIRECTION_MASK;
702
703         destroy_cmd.sq.sq_idx = io_sq->idx;
704         destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_SQ;
705
706         ret = ena_com_execute_admin_command(admin_queue,
707                                             (struct ena_admin_aq_entry *)&destroy_cmd,
708                                             sizeof(destroy_cmd),
709                                             (struct ena_admin_acq_entry *)&destroy_resp,
710                                             sizeof(destroy_resp));
711
712         if (unlikely(ret && (ret != -ENODEV)))
713                 pr_err("failed to destroy io sq error: %d\n", ret);
714
715         return ret;
716 }
717
718 static void ena_com_io_queue_free(struct ena_com_dev *ena_dev,
719                                   struct ena_com_io_sq *io_sq,
720                                   struct ena_com_io_cq *io_cq)
721 {
722         size_t size;
723
724         if (io_cq->cdesc_addr.virt_addr) {
725                 size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
726
727                 dma_free_coherent(ena_dev->dmadev, size,
728                                   io_cq->cdesc_addr.virt_addr,
729                                   io_cq->cdesc_addr.phys_addr);
730
731                 io_cq->cdesc_addr.virt_addr = NULL;
732         }
733
734         if (io_sq->desc_addr.virt_addr) {
735                 size = io_sq->desc_entry_size * io_sq->q_depth;
736
737                 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
738                         dma_free_coherent(ena_dev->dmadev, size,
739                                           io_sq->desc_addr.virt_addr,
740                                           io_sq->desc_addr.phys_addr);
741                 else
742                         devm_kfree(ena_dev->dmadev, io_sq->desc_addr.virt_addr);
743
744                 io_sq->desc_addr.virt_addr = NULL;
745         }
746 }
747
748 static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout,
749                                 u16 exp_state)
750 {
751         u32 val, i;
752
753         /* Convert timeout from resolution of 100ms to ENA_POLL_MS */
754         timeout = (timeout * 100) / ENA_POLL_MS;
755
756         for (i = 0; i < timeout; i++) {
757                 val = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
758
759                 if (unlikely(val == ENA_MMIO_READ_TIMEOUT)) {
760                         pr_err("Reg read timeout occurred\n");
761                         return -ETIME;
762                 }
763
764                 if ((val & ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK) ==
765                         exp_state)
766                         return 0;
767
768                 msleep(ENA_POLL_MS);
769         }
770
771         return -ETIME;
772 }
773
774 static bool ena_com_check_supported_feature_id(struct ena_com_dev *ena_dev,
775                                                enum ena_admin_aq_feature_id feature_id)
776 {
777         u32 feature_mask = 1 << feature_id;
778
779         /* Device attributes is always supported */
780         if ((feature_id != ENA_ADMIN_DEVICE_ATTRIBUTES) &&
781             !(ena_dev->supported_features & feature_mask))
782                 return false;
783
784         return true;
785 }
786
787 static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev,
788                                   struct ena_admin_get_feat_resp *get_resp,
789                                   enum ena_admin_aq_feature_id feature_id,
790                                   dma_addr_t control_buf_dma_addr,
791                                   u32 control_buff_size)
792 {
793         struct ena_com_admin_queue *admin_queue;
794         struct ena_admin_get_feat_cmd get_cmd;
795         int ret;
796
797         if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) {
798                 pr_debug("Feature %d isn't supported\n", feature_id);
799                 return -EOPNOTSUPP;
800         }
801
802         memset(&get_cmd, 0x0, sizeof(get_cmd));
803         admin_queue = &ena_dev->admin_queue;
804
805         get_cmd.aq_common_descriptor.opcode = ENA_ADMIN_GET_FEATURE;
806
807         if (control_buff_size)
808                 get_cmd.aq_common_descriptor.flags =
809                         ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
810         else
811                 get_cmd.aq_common_descriptor.flags = 0;
812
813         ret = ena_com_mem_addr_set(ena_dev,
814                                    &get_cmd.control_buffer.address,
815                                    control_buf_dma_addr);
816         if (unlikely(ret)) {
817                 pr_err("memory address set failed\n");
818                 return ret;
819         }
820
821         get_cmd.control_buffer.length = control_buff_size;
822
823         get_cmd.feat_common.feature_id = feature_id;
824
825         ret = ena_com_execute_admin_command(admin_queue,
826                                             (struct ena_admin_aq_entry *)
827                                             &get_cmd,
828                                             sizeof(get_cmd),
829                                             (struct ena_admin_acq_entry *)
830                                             get_resp,
831                                             sizeof(*get_resp));
832
833         if (unlikely(ret))
834                 pr_err("Failed to submit get_feature command %d error: %d\n",
835                        feature_id, ret);
836
837         return ret;
838 }
839
840 static int ena_com_get_feature(struct ena_com_dev *ena_dev,
841                                struct ena_admin_get_feat_resp *get_resp,
842                                enum ena_admin_aq_feature_id feature_id)
843 {
844         return ena_com_get_feature_ex(ena_dev,
845                                       get_resp,
846                                       feature_id,
847                                       0,
848                                       0);
849 }
850
851 static int ena_com_hash_key_allocate(struct ena_com_dev *ena_dev)
852 {
853         struct ena_rss *rss = &ena_dev->rss;
854
855         rss->hash_key =
856                 dma_zalloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_key),
857                                     &rss->hash_key_dma_addr, GFP_KERNEL);
858
859         if (unlikely(!rss->hash_key))
860                 return -ENOMEM;
861
862         return 0;
863 }
864
865 static void ena_com_hash_key_destroy(struct ena_com_dev *ena_dev)
866 {
867         struct ena_rss *rss = &ena_dev->rss;
868
869         if (rss->hash_key)
870                 dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_key),
871                                   rss->hash_key, rss->hash_key_dma_addr);
872         rss->hash_key = NULL;
873 }
874
875 static int ena_com_hash_ctrl_init(struct ena_com_dev *ena_dev)
876 {
877         struct ena_rss *rss = &ena_dev->rss;
878
879         rss->hash_ctrl =
880                 dma_zalloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl),
881                                     &rss->hash_ctrl_dma_addr, GFP_KERNEL);
882
883         if (unlikely(!rss->hash_ctrl))
884                 return -ENOMEM;
885
886         return 0;
887 }
888
889 static void ena_com_hash_ctrl_destroy(struct ena_com_dev *ena_dev)
890 {
891         struct ena_rss *rss = &ena_dev->rss;
892
893         if (rss->hash_ctrl)
894                 dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl),
895                                   rss->hash_ctrl, rss->hash_ctrl_dma_addr);
896         rss->hash_ctrl = NULL;
897 }
898
899 static int ena_com_indirect_table_allocate(struct ena_com_dev *ena_dev,
900                                            u16 log_size)
901 {
902         struct ena_rss *rss = &ena_dev->rss;
903         struct ena_admin_get_feat_resp get_resp;
904         size_t tbl_size;
905         int ret;
906
907         ret = ena_com_get_feature(ena_dev, &get_resp,
908                                   ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
909         if (unlikely(ret))
910                 return ret;
911
912         if ((get_resp.u.ind_table.min_size > log_size) ||
913             (get_resp.u.ind_table.max_size < log_size)) {
914                 pr_err("indirect table size doesn't fit. requested size: %d while min is:%d and max %d\n",
915                        1 << log_size, 1 << get_resp.u.ind_table.min_size,
916                        1 << get_resp.u.ind_table.max_size);
917                 return -EINVAL;
918         }
919
920         tbl_size = (1ULL << log_size) *
921                 sizeof(struct ena_admin_rss_ind_table_entry);
922
923         rss->rss_ind_tbl =
924                 dma_zalloc_coherent(ena_dev->dmadev, tbl_size,
925                                     &rss->rss_ind_tbl_dma_addr, GFP_KERNEL);
926         if (unlikely(!rss->rss_ind_tbl))
927                 goto mem_err1;
928
929         tbl_size = (1ULL << log_size) * sizeof(u16);
930         rss->host_rss_ind_tbl =
931                 devm_kzalloc(ena_dev->dmadev, tbl_size, GFP_KERNEL);
932         if (unlikely(!rss->host_rss_ind_tbl))
933                 goto mem_err2;
934
935         rss->tbl_log_size = log_size;
936
937         return 0;
938
939 mem_err2:
940         tbl_size = (1ULL << log_size) *
941                 sizeof(struct ena_admin_rss_ind_table_entry);
942
943         dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl,
944                           rss->rss_ind_tbl_dma_addr);
945         rss->rss_ind_tbl = NULL;
946 mem_err1:
947         rss->tbl_log_size = 0;
948         return -ENOMEM;
949 }
950
951 static void ena_com_indirect_table_destroy(struct ena_com_dev *ena_dev)
952 {
953         struct ena_rss *rss = &ena_dev->rss;
954         size_t tbl_size = (1ULL << rss->tbl_log_size) *
955                 sizeof(struct ena_admin_rss_ind_table_entry);
956
957         if (rss->rss_ind_tbl)
958                 dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl,
959                                   rss->rss_ind_tbl_dma_addr);
960         rss->rss_ind_tbl = NULL;
961
962         if (rss->host_rss_ind_tbl)
963                 devm_kfree(ena_dev->dmadev, rss->host_rss_ind_tbl);
964         rss->host_rss_ind_tbl = NULL;
965 }
966
967 static int ena_com_create_io_sq(struct ena_com_dev *ena_dev,
968                                 struct ena_com_io_sq *io_sq, u16 cq_idx)
969 {
970         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
971         struct ena_admin_aq_create_sq_cmd create_cmd;
972         struct ena_admin_acq_create_sq_resp_desc cmd_completion;
973         u8 direction;
974         int ret;
975
976         memset(&create_cmd, 0x0, sizeof(create_cmd));
977
978         create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_SQ;
979
980         if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
981                 direction = ENA_ADMIN_SQ_DIRECTION_TX;
982         else
983                 direction = ENA_ADMIN_SQ_DIRECTION_RX;
984
985         create_cmd.sq_identity |= (direction <<
986                 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT) &
987                 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK;
988
989         create_cmd.sq_caps_2 |= io_sq->mem_queue_type &
990                 ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK;
991
992         create_cmd.sq_caps_2 |= (ENA_ADMIN_COMPLETION_POLICY_DESC <<
993                 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT) &
994                 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK;
995
996         create_cmd.sq_caps_3 |=
997                 ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK;
998
999         create_cmd.cq_idx = cq_idx;
1000         create_cmd.sq_depth = io_sq->q_depth;
1001
1002         if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) {
1003                 ret = ena_com_mem_addr_set(ena_dev,
1004                                            &create_cmd.sq_ba,
1005                                            io_sq->desc_addr.phys_addr);
1006                 if (unlikely(ret)) {
1007                         pr_err("memory address set failed\n");
1008                         return ret;
1009                 }
1010         }
1011
1012         ret = ena_com_execute_admin_command(admin_queue,
1013                                             (struct ena_admin_aq_entry *)&create_cmd,
1014                                             sizeof(create_cmd),
1015                                             (struct ena_admin_acq_entry *)&cmd_completion,
1016                                             sizeof(cmd_completion));
1017         if (unlikely(ret)) {
1018                 pr_err("Failed to create IO SQ. error: %d\n", ret);
1019                 return ret;
1020         }
1021
1022         io_sq->idx = cmd_completion.sq_idx;
1023
1024         io_sq->db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1025                 (uintptr_t)cmd_completion.sq_doorbell_offset);
1026
1027         if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
1028                 io_sq->header_addr = (u8 __iomem *)((uintptr_t)ena_dev->mem_bar
1029                                 + cmd_completion.llq_headers_offset);
1030
1031                 io_sq->desc_addr.pbuf_dev_addr =
1032                         (u8 __iomem *)((uintptr_t)ena_dev->mem_bar +
1033                         cmd_completion.llq_descriptors_offset);
1034         }
1035
1036         pr_debug("created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth);
1037
1038         return ret;
1039 }
1040
1041 static int ena_com_ind_tbl_convert_to_device(struct ena_com_dev *ena_dev)
1042 {
1043         struct ena_rss *rss = &ena_dev->rss;
1044         struct ena_com_io_sq *io_sq;
1045         u16 qid;
1046         int i;
1047
1048         for (i = 0; i < 1 << rss->tbl_log_size; i++) {
1049                 qid = rss->host_rss_ind_tbl[i];
1050                 if (qid >= ENA_TOTAL_NUM_QUEUES)
1051                         return -EINVAL;
1052
1053                 io_sq = &ena_dev->io_sq_queues[qid];
1054
1055                 if (io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX)
1056                         return -EINVAL;
1057
1058                 rss->rss_ind_tbl[i].cq_idx = io_sq->idx;
1059         }
1060
1061         return 0;
1062 }
1063
1064 static int ena_com_ind_tbl_convert_from_device(struct ena_com_dev *ena_dev)
1065 {
1066         u16 dev_idx_to_host_tbl[ENA_TOTAL_NUM_QUEUES] = { (u16)-1 };
1067         struct ena_rss *rss = &ena_dev->rss;
1068         u8 idx;
1069         u16 i;
1070
1071         for (i = 0; i < ENA_TOTAL_NUM_QUEUES; i++)
1072                 dev_idx_to_host_tbl[ena_dev->io_sq_queues[i].idx] = i;
1073
1074         for (i = 0; i < 1 << rss->tbl_log_size; i++) {
1075                 if (rss->rss_ind_tbl[i].cq_idx > ENA_TOTAL_NUM_QUEUES)
1076                         return -EINVAL;
1077                 idx = (u8)rss->rss_ind_tbl[i].cq_idx;
1078
1079                 if (dev_idx_to_host_tbl[idx] > ENA_TOTAL_NUM_QUEUES)
1080                         return -EINVAL;
1081
1082                 rss->host_rss_ind_tbl[i] = dev_idx_to_host_tbl[idx];
1083         }
1084
1085         return 0;
1086 }
1087
1088 static int ena_com_init_interrupt_moderation_table(struct ena_com_dev *ena_dev)
1089 {
1090         size_t size;
1091
1092         size = sizeof(struct ena_intr_moder_entry) * ENA_INTR_MAX_NUM_OF_LEVELS;
1093
1094         ena_dev->intr_moder_tbl =
1095                 devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
1096         if (!ena_dev->intr_moder_tbl)
1097                 return -ENOMEM;
1098
1099         ena_com_config_default_interrupt_moderation_table(ena_dev);
1100
1101         return 0;
1102 }
1103
1104 static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev,
1105                                                  u16 intr_delay_resolution)
1106 {
1107         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
1108         unsigned int i;
1109
1110         if (!intr_delay_resolution) {
1111                 pr_err("Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n");
1112                 intr_delay_resolution = 1;
1113         }
1114         ena_dev->intr_delay_resolution = intr_delay_resolution;
1115
1116         /* update Rx */
1117         for (i = 0; i < ENA_INTR_MAX_NUM_OF_LEVELS; i++)
1118                 intr_moder_tbl[i].intr_moder_interval /= intr_delay_resolution;
1119
1120         /* update Tx */
1121         ena_dev->intr_moder_tx_interval /= intr_delay_resolution;
1122 }
1123
1124 /*****************************************************************************/
1125 /*******************************      API       ******************************/
1126 /*****************************************************************************/
1127
1128 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,
1129                                   struct ena_admin_aq_entry *cmd,
1130                                   size_t cmd_size,
1131                                   struct ena_admin_acq_entry *comp,
1132                                   size_t comp_size)
1133 {
1134         struct ena_comp_ctx *comp_ctx;
1135         int ret;
1136
1137         comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,
1138                                             comp, comp_size);
1139         if (IS_ERR(comp_ctx)) {
1140                 if (comp_ctx == ERR_PTR(-ENODEV))
1141                         pr_debug("Failed to submit command [%ld]\n",
1142                                  PTR_ERR(comp_ctx));
1143                 else
1144                         pr_err("Failed to submit command [%ld]\n",
1145                                PTR_ERR(comp_ctx));
1146
1147                 return PTR_ERR(comp_ctx);
1148         }
1149
1150         ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);
1151         if (unlikely(ret)) {
1152                 if (admin_queue->running_state)
1153                         pr_err("Failed to process command. ret = %d\n", ret);
1154                 else
1155                         pr_debug("Failed to process command. ret = %d\n", ret);
1156         }
1157         return ret;
1158 }
1159
1160 int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
1161                          struct ena_com_io_cq *io_cq)
1162 {
1163         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1164         struct ena_admin_aq_create_cq_cmd create_cmd;
1165         struct ena_admin_acq_create_cq_resp_desc cmd_completion;
1166         int ret;
1167
1168         memset(&create_cmd, 0x0, sizeof(create_cmd));
1169
1170         create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_CQ;
1171
1172         create_cmd.cq_caps_2 |= (io_cq->cdesc_entry_size_in_bytes / 4) &
1173                 ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;
1174         create_cmd.cq_caps_1 |=
1175                 ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK;
1176
1177         create_cmd.msix_vector = io_cq->msix_vector;
1178         create_cmd.cq_depth = io_cq->q_depth;
1179
1180         ret = ena_com_mem_addr_set(ena_dev,
1181                                    &create_cmd.cq_ba,
1182                                    io_cq->cdesc_addr.phys_addr);
1183         if (unlikely(ret)) {
1184                 pr_err("memory address set failed\n");
1185                 return ret;
1186         }
1187
1188         ret = ena_com_execute_admin_command(admin_queue,
1189                                             (struct ena_admin_aq_entry *)&create_cmd,
1190                                             sizeof(create_cmd),
1191                                             (struct ena_admin_acq_entry *)&cmd_completion,
1192                                             sizeof(cmd_completion));
1193         if (unlikely(ret)) {
1194                 pr_err("Failed to create IO CQ. error: %d\n", ret);
1195                 return ret;
1196         }
1197
1198         io_cq->idx = cmd_completion.cq_idx;
1199
1200         io_cq->unmask_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1201                 cmd_completion.cq_interrupt_unmask_register_offset);
1202
1203         if (cmd_completion.cq_head_db_register_offset)
1204                 io_cq->cq_head_db_reg =
1205                         (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1206                         cmd_completion.cq_head_db_register_offset);
1207
1208         if (cmd_completion.numa_node_register_offset)
1209                 io_cq->numa_node_cfg_reg =
1210                         (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1211                         cmd_completion.numa_node_register_offset);
1212
1213         pr_debug("created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth);
1214
1215         return ret;
1216 }
1217
1218 int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid,
1219                             struct ena_com_io_sq **io_sq,
1220                             struct ena_com_io_cq **io_cq)
1221 {
1222         if (qid >= ENA_TOTAL_NUM_QUEUES) {
1223                 pr_err("Invalid queue number %d but the max is %d\n", qid,
1224                        ENA_TOTAL_NUM_QUEUES);
1225                 return -EINVAL;
1226         }
1227
1228         *io_sq = &ena_dev->io_sq_queues[qid];
1229         *io_cq = &ena_dev->io_cq_queues[qid];
1230
1231         return 0;
1232 }
1233
1234 void ena_com_abort_admin_commands(struct ena_com_dev *ena_dev)
1235 {
1236         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1237         struct ena_comp_ctx *comp_ctx;
1238         u16 i;
1239
1240         if (!admin_queue->comp_ctx)
1241                 return;
1242
1243         for (i = 0; i < admin_queue->q_depth; i++) {
1244                 comp_ctx = get_comp_ctxt(admin_queue, i, false);
1245                 if (unlikely(!comp_ctx))
1246                         break;
1247
1248                 comp_ctx->status = ENA_CMD_ABORTED;
1249
1250                 complete(&comp_ctx->wait_event);
1251         }
1252 }
1253
1254 void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev)
1255 {
1256         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1257         unsigned long flags;
1258
1259         spin_lock_irqsave(&admin_queue->q_lock, flags);
1260         while (atomic_read(&admin_queue->outstanding_cmds) != 0) {
1261                 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1262                 msleep(ENA_POLL_MS);
1263                 spin_lock_irqsave(&admin_queue->q_lock, flags);
1264         }
1265         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1266 }
1267
1268 int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev,
1269                           struct ena_com_io_cq *io_cq)
1270 {
1271         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1272         struct ena_admin_aq_destroy_cq_cmd destroy_cmd;
1273         struct ena_admin_acq_destroy_cq_resp_desc destroy_resp;
1274         int ret;
1275
1276         memset(&destroy_cmd, 0x0, sizeof(destroy_cmd));
1277
1278         destroy_cmd.cq_idx = io_cq->idx;
1279         destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_CQ;
1280
1281         ret = ena_com_execute_admin_command(admin_queue,
1282                                             (struct ena_admin_aq_entry *)&destroy_cmd,
1283                                             sizeof(destroy_cmd),
1284                                             (struct ena_admin_acq_entry *)&destroy_resp,
1285                                             sizeof(destroy_resp));
1286
1287         if (unlikely(ret && (ret != -ENODEV)))
1288                 pr_err("Failed to destroy IO CQ. error: %d\n", ret);
1289
1290         return ret;
1291 }
1292
1293 bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev)
1294 {
1295         return ena_dev->admin_queue.running_state;
1296 }
1297
1298 void ena_com_set_admin_running_state(struct ena_com_dev *ena_dev, bool state)
1299 {
1300         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1301         unsigned long flags;
1302
1303         spin_lock_irqsave(&admin_queue->q_lock, flags);
1304         ena_dev->admin_queue.running_state = state;
1305         spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1306 }
1307
1308 void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev)
1309 {
1310         u16 depth = ena_dev->aenq.q_depth;
1311
1312         WARN(ena_dev->aenq.head != depth, "Invalid AENQ state\n");
1313
1314         /* Init head_db to mark that all entries in the queue
1315          * are initially available
1316          */
1317         writel(depth, ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
1318 }
1319
1320 int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
1321 {
1322         struct ena_com_admin_queue *admin_queue;
1323         struct ena_admin_set_feat_cmd cmd;
1324         struct ena_admin_set_feat_resp resp;
1325         struct ena_admin_get_feat_resp get_resp;
1326         int ret;
1327
1328         ret = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_AENQ_CONFIG);
1329         if (ret) {
1330                 pr_info("Can't get aenq configuration\n");
1331                 return ret;
1332         }
1333
1334         if ((get_resp.u.aenq.supported_groups & groups_flag) != groups_flag) {
1335                 pr_warn("Trying to set unsupported aenq events. supported flag: %x asked flag: %x\n",
1336                         get_resp.u.aenq.supported_groups, groups_flag);
1337                 return -EOPNOTSUPP;
1338         }
1339
1340         memset(&cmd, 0x0, sizeof(cmd));
1341         admin_queue = &ena_dev->admin_queue;
1342
1343         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
1344         cmd.aq_common_descriptor.flags = 0;
1345         cmd.feat_common.feature_id = ENA_ADMIN_AENQ_CONFIG;
1346         cmd.u.aenq.enabled_groups = groups_flag;
1347
1348         ret = ena_com_execute_admin_command(admin_queue,
1349                                             (struct ena_admin_aq_entry *)&cmd,
1350                                             sizeof(cmd),
1351                                             (struct ena_admin_acq_entry *)&resp,
1352                                             sizeof(resp));
1353
1354         if (unlikely(ret))
1355                 pr_err("Failed to config AENQ ret: %d\n", ret);
1356
1357         return ret;
1358 }
1359
1360 int ena_com_get_dma_width(struct ena_com_dev *ena_dev)
1361 {
1362         u32 caps = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
1363         int width;
1364
1365         if (unlikely(caps == ENA_MMIO_READ_TIMEOUT)) {
1366                 pr_err("Reg read timeout occurred\n");
1367                 return -ETIME;
1368         }
1369
1370         width = (caps & ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) >>
1371                 ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT;
1372
1373         pr_debug("ENA dma width: %d\n", width);
1374
1375         if ((width < 32) || width > ENA_MAX_PHYS_ADDR_SIZE_BITS) {
1376                 pr_err("DMA width illegal value: %d\n", width);
1377                 return -EINVAL;
1378         }
1379
1380         ena_dev->dma_addr_bits = width;
1381
1382         return width;
1383 }
1384
1385 int ena_com_validate_version(struct ena_com_dev *ena_dev)
1386 {
1387         u32 ver;
1388         u32 ctrl_ver;
1389         u32 ctrl_ver_masked;
1390
1391         /* Make sure the ENA version and the controller version are at least
1392          * as the driver expects
1393          */
1394         ver = ena_com_reg_bar_read32(ena_dev, ENA_REGS_VERSION_OFF);
1395         ctrl_ver = ena_com_reg_bar_read32(ena_dev,
1396                                           ENA_REGS_CONTROLLER_VERSION_OFF);
1397
1398         if (unlikely((ver == ENA_MMIO_READ_TIMEOUT) ||
1399                      (ctrl_ver == ENA_MMIO_READ_TIMEOUT))) {
1400                 pr_err("Reg read timeout occurred\n");
1401                 return -ETIME;
1402         }
1403
1404         pr_info("ena device version: %d.%d\n",
1405                 (ver & ENA_REGS_VERSION_MAJOR_VERSION_MASK) >>
1406                         ENA_REGS_VERSION_MAJOR_VERSION_SHIFT,
1407                 ver & ENA_REGS_VERSION_MINOR_VERSION_MASK);
1408
1409         if (ver < MIN_ENA_VER) {
1410                 pr_err("ENA version is lower than the minimal version the driver supports\n");
1411                 return -1;
1412         }
1413
1414         pr_info("ena controller version: %d.%d.%d implementation version %d\n",
1415                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) >>
1416                         ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT,
1417                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) >>
1418                         ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT,
1419                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK),
1420                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK) >>
1421                         ENA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT);
1422
1423         ctrl_ver_masked =
1424                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) |
1425                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) |
1426                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK);
1427
1428         /* Validate the ctrl version without the implementation ID */
1429         if (ctrl_ver_masked < MIN_ENA_CTRL_VER) {
1430                 pr_err("ENA ctrl version is lower than the minimal ctrl version the driver supports\n");
1431                 return -1;
1432         }
1433
1434         return 0;
1435 }
1436
1437 void ena_com_admin_destroy(struct ena_com_dev *ena_dev)
1438 {
1439         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1440         struct ena_com_admin_cq *cq = &admin_queue->cq;
1441         struct ena_com_admin_sq *sq = &admin_queue->sq;
1442         struct ena_com_aenq *aenq = &ena_dev->aenq;
1443         u16 size;
1444
1445         if (admin_queue->comp_ctx)
1446                 devm_kfree(ena_dev->dmadev, admin_queue->comp_ctx);
1447         admin_queue->comp_ctx = NULL;
1448         size = ADMIN_SQ_SIZE(admin_queue->q_depth);
1449         if (sq->entries)
1450                 dma_free_coherent(ena_dev->dmadev, size, sq->entries,
1451                                   sq->dma_addr);
1452         sq->entries = NULL;
1453
1454         size = ADMIN_CQ_SIZE(admin_queue->q_depth);
1455         if (cq->entries)
1456                 dma_free_coherent(ena_dev->dmadev, size, cq->entries,
1457                                   cq->dma_addr);
1458         cq->entries = NULL;
1459
1460         size = ADMIN_AENQ_SIZE(aenq->q_depth);
1461         if (ena_dev->aenq.entries)
1462                 dma_free_coherent(ena_dev->dmadev, size, aenq->entries,
1463                                   aenq->dma_addr);
1464         aenq->entries = NULL;
1465 }
1466
1467 void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling)
1468 {
1469         u32 mask_value = 0;
1470
1471         if (polling)
1472                 mask_value = ENA_REGS_ADMIN_INTR_MASK;
1473
1474         writel(mask_value, ena_dev->reg_bar + ENA_REGS_INTR_MASK_OFF);
1475         ena_dev->admin_queue.polling = polling;
1476 }
1477
1478 int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev)
1479 {
1480         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1481
1482         spin_lock_init(&mmio_read->lock);
1483         mmio_read->read_resp =
1484                 dma_zalloc_coherent(ena_dev->dmadev,
1485                                     sizeof(*mmio_read->read_resp),
1486                                     &mmio_read->read_resp_dma_addr, GFP_KERNEL);
1487         if (unlikely(!mmio_read->read_resp))
1488                 return -ENOMEM;
1489
1490         ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
1491
1492         mmio_read->read_resp->req_id = 0x0;
1493         mmio_read->seq_num = 0x0;
1494         mmio_read->readless_supported = true;
1495
1496         return 0;
1497 }
1498
1499 void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported)
1500 {
1501         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1502
1503         mmio_read->readless_supported = readless_supported;
1504 }
1505
1506 void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev)
1507 {
1508         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1509
1510         writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
1511         writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
1512
1513         dma_free_coherent(ena_dev->dmadev, sizeof(*mmio_read->read_resp),
1514                           mmio_read->read_resp, mmio_read->read_resp_dma_addr);
1515
1516         mmio_read->read_resp = NULL;
1517 }
1518
1519 void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev)
1520 {
1521         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1522         u32 addr_low, addr_high;
1523
1524         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(mmio_read->read_resp_dma_addr);
1525         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(mmio_read->read_resp_dma_addr);
1526
1527         writel(addr_low, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
1528         writel(addr_high, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
1529 }
1530
1531 int ena_com_admin_init(struct ena_com_dev *ena_dev,
1532                        struct ena_aenq_handlers *aenq_handlers,
1533                        bool init_spinlock)
1534 {
1535         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1536         u32 aq_caps, acq_caps, dev_sts, addr_low, addr_high;
1537         int ret;
1538
1539         dev_sts = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
1540
1541         if (unlikely(dev_sts == ENA_MMIO_READ_TIMEOUT)) {
1542                 pr_err("Reg read timeout occurred\n");
1543                 return -ETIME;
1544         }
1545
1546         if (!(dev_sts & ENA_REGS_DEV_STS_READY_MASK)) {
1547                 pr_err("Device isn't ready, abort com init\n");
1548                 return -ENODEV;
1549         }
1550
1551         admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;
1552
1553         admin_queue->q_dmadev = ena_dev->dmadev;
1554         admin_queue->polling = false;
1555         admin_queue->curr_cmd_id = 0;
1556
1557         atomic_set(&admin_queue->outstanding_cmds, 0);
1558
1559         if (init_spinlock)
1560                 spin_lock_init(&admin_queue->q_lock);
1561
1562         ret = ena_com_init_comp_ctxt(admin_queue);
1563         if (ret)
1564                 goto error;
1565
1566         ret = ena_com_admin_init_sq(admin_queue);
1567         if (ret)
1568                 goto error;
1569
1570         ret = ena_com_admin_init_cq(admin_queue);
1571         if (ret)
1572                 goto error;
1573
1574         admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1575                 ENA_REGS_AQ_DB_OFF);
1576
1577         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);
1578         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);
1579
1580         writel(addr_low, ena_dev->reg_bar + ENA_REGS_AQ_BASE_LO_OFF);
1581         writel(addr_high, ena_dev->reg_bar + ENA_REGS_AQ_BASE_HI_OFF);
1582
1583         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);
1584         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);
1585
1586         writel(addr_low, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_LO_OFF);
1587         writel(addr_high, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_HI_OFF);
1588
1589         aq_caps = 0;
1590         aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;
1591         aq_caps |= (sizeof(struct ena_admin_aq_entry) <<
1592                         ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT) &
1593                         ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK;
1594
1595         acq_caps = 0;
1596         acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;
1597         acq_caps |= (sizeof(struct ena_admin_acq_entry) <<
1598                 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) &
1599                 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK;
1600
1601         writel(aq_caps, ena_dev->reg_bar + ENA_REGS_AQ_CAPS_OFF);
1602         writel(acq_caps, ena_dev->reg_bar + ENA_REGS_ACQ_CAPS_OFF);
1603         ret = ena_com_admin_init_aenq(ena_dev, aenq_handlers);
1604         if (ret)
1605                 goto error;
1606
1607         admin_queue->running_state = true;
1608
1609         return 0;
1610 error:
1611         ena_com_admin_destroy(ena_dev);
1612
1613         return ret;
1614 }
1615
1616 int ena_com_create_io_queue(struct ena_com_dev *ena_dev,
1617                             struct ena_com_create_io_ctx *ctx)
1618 {
1619         struct ena_com_io_sq *io_sq;
1620         struct ena_com_io_cq *io_cq;
1621         int ret;
1622
1623         if (ctx->qid >= ENA_TOTAL_NUM_QUEUES) {
1624                 pr_err("Qid (%d) is bigger than max num of queues (%d)\n",
1625                        ctx->qid, ENA_TOTAL_NUM_QUEUES);
1626                 return -EINVAL;
1627         }
1628
1629         io_sq = &ena_dev->io_sq_queues[ctx->qid];
1630         io_cq = &ena_dev->io_cq_queues[ctx->qid];
1631
1632         memset(io_sq, 0x0, sizeof(*io_sq));
1633         memset(io_cq, 0x0, sizeof(*io_cq));
1634
1635         /* Init CQ */
1636         io_cq->q_depth = ctx->queue_size;
1637         io_cq->direction = ctx->direction;
1638         io_cq->qid = ctx->qid;
1639
1640         io_cq->msix_vector = ctx->msix_vector;
1641
1642         io_sq->q_depth = ctx->queue_size;
1643         io_sq->direction = ctx->direction;
1644         io_sq->qid = ctx->qid;
1645
1646         io_sq->mem_queue_type = ctx->mem_queue_type;
1647
1648         if (ctx->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
1649                 /* header length is limited to 8 bits */
1650                 io_sq->tx_max_header_size =
1651                         min_t(u32, ena_dev->tx_max_header_size, SZ_256);
1652
1653         ret = ena_com_init_io_sq(ena_dev, ctx, io_sq);
1654         if (ret)
1655                 goto error;
1656         ret = ena_com_init_io_cq(ena_dev, ctx, io_cq);
1657         if (ret)
1658                 goto error;
1659
1660         ret = ena_com_create_io_cq(ena_dev, io_cq);
1661         if (ret)
1662                 goto error;
1663
1664         ret = ena_com_create_io_sq(ena_dev, io_sq, io_cq->idx);
1665         if (ret)
1666                 goto destroy_io_cq;
1667
1668         return 0;
1669
1670 destroy_io_cq:
1671         ena_com_destroy_io_cq(ena_dev, io_cq);
1672 error:
1673         ena_com_io_queue_free(ena_dev, io_sq, io_cq);
1674         return ret;
1675 }
1676
1677 void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid)
1678 {
1679         struct ena_com_io_sq *io_sq;
1680         struct ena_com_io_cq *io_cq;
1681
1682         if (qid >= ENA_TOTAL_NUM_QUEUES) {
1683                 pr_err("Qid (%d) is bigger than max num of queues (%d)\n", qid,
1684                        ENA_TOTAL_NUM_QUEUES);
1685                 return;
1686         }
1687
1688         io_sq = &ena_dev->io_sq_queues[qid];
1689         io_cq = &ena_dev->io_cq_queues[qid];
1690
1691         ena_com_destroy_io_sq(ena_dev, io_sq);
1692         ena_com_destroy_io_cq(ena_dev, io_cq);
1693
1694         ena_com_io_queue_free(ena_dev, io_sq, io_cq);
1695 }
1696
1697 int ena_com_get_link_params(struct ena_com_dev *ena_dev,
1698                             struct ena_admin_get_feat_resp *resp)
1699 {
1700         return ena_com_get_feature(ena_dev, resp, ENA_ADMIN_LINK_CONFIG);
1701 }
1702
1703 int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
1704                               struct ena_com_dev_get_features_ctx *get_feat_ctx)
1705 {
1706         struct ena_admin_get_feat_resp get_resp;
1707         int rc;
1708
1709         rc = ena_com_get_feature(ena_dev, &get_resp,
1710                                  ENA_ADMIN_DEVICE_ATTRIBUTES);
1711         if (rc)
1712                 return rc;
1713
1714         memcpy(&get_feat_ctx->dev_attr, &get_resp.u.dev_attr,
1715                sizeof(get_resp.u.dev_attr));
1716         ena_dev->supported_features = get_resp.u.dev_attr.supported_features;
1717
1718         rc = ena_com_get_feature(ena_dev, &get_resp,
1719                                  ENA_ADMIN_MAX_QUEUES_NUM);
1720         if (rc)
1721                 return rc;
1722
1723         memcpy(&get_feat_ctx->max_queues, &get_resp.u.max_queue,
1724                sizeof(get_resp.u.max_queue));
1725         ena_dev->tx_max_header_size = get_resp.u.max_queue.max_header_size;
1726
1727         rc = ena_com_get_feature(ena_dev, &get_resp,
1728                                  ENA_ADMIN_AENQ_CONFIG);
1729         if (rc)
1730                 return rc;
1731
1732         memcpy(&get_feat_ctx->aenq, &get_resp.u.aenq,
1733                sizeof(get_resp.u.aenq));
1734
1735         rc = ena_com_get_feature(ena_dev, &get_resp,
1736                                  ENA_ADMIN_STATELESS_OFFLOAD_CONFIG);
1737         if (rc)
1738                 return rc;
1739
1740         memcpy(&get_feat_ctx->offload, &get_resp.u.offload,
1741                sizeof(get_resp.u.offload));
1742
1743         /* Driver hints isn't mandatory admin command. So in case the
1744          * command isn't supported set driver hints to 0
1745          */
1746         rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_HW_HINTS);
1747
1748         if (!rc)
1749                 memcpy(&get_feat_ctx->hw_hints, &get_resp.u.hw_hints,
1750                        sizeof(get_resp.u.hw_hints));
1751         else if (rc == -EOPNOTSUPP)
1752                 memset(&get_feat_ctx->hw_hints, 0x0,
1753                        sizeof(get_feat_ctx->hw_hints));
1754         else
1755                 return rc;
1756
1757         return 0;
1758 }
1759
1760 void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev)
1761 {
1762         ena_com_handle_admin_completion(&ena_dev->admin_queue);
1763 }
1764
1765 /* ena_handle_specific_aenq_event:
1766  * return the handler that is relevant to the specific event group
1767  */
1768 static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *dev,
1769                                                      u16 group)
1770 {
1771         struct ena_aenq_handlers *aenq_handlers = dev->aenq.aenq_handlers;
1772
1773         if ((group < ENA_MAX_HANDLERS) && aenq_handlers->handlers[group])
1774                 return aenq_handlers->handlers[group];
1775
1776         return aenq_handlers->unimplemented_handler;
1777 }
1778
1779 /* ena_aenq_intr_handler:
1780  * handles the aenq incoming events.
1781  * pop events from the queue and apply the specific handler
1782  */
1783 void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
1784 {
1785         struct ena_admin_aenq_entry *aenq_e;
1786         struct ena_admin_aenq_common_desc *aenq_common;
1787         struct ena_com_aenq *aenq  = &dev->aenq;
1788         ena_aenq_handler handler_cb;
1789         u16 masked_head, processed = 0;
1790         u8 phase;
1791
1792         masked_head = aenq->head & (aenq->q_depth - 1);
1793         phase = aenq->phase;
1794         aenq_e = &aenq->entries[masked_head]; /* Get first entry */
1795         aenq_common = &aenq_e->aenq_common_desc;
1796
1797         /* Go over all the events */
1798         while ((aenq_common->flags & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) ==
1799                phase) {
1800                 pr_debug("AENQ! Group[%x] Syndrom[%x] timestamp: [%llus]\n",
1801                          aenq_common->group, aenq_common->syndrom,
1802                          (u64)aenq_common->timestamp_low +
1803                                  ((u64)aenq_common->timestamp_high << 32));
1804
1805                 /* Handle specific event*/
1806                 handler_cb = ena_com_get_specific_aenq_cb(dev,
1807                                                           aenq_common->group);
1808                 handler_cb(data, aenq_e); /* call the actual event handler*/
1809
1810                 /* Get next event entry */
1811                 masked_head++;
1812                 processed++;
1813
1814                 if (unlikely(masked_head == aenq->q_depth)) {
1815                         masked_head = 0;
1816                         phase = !phase;
1817                 }
1818                 aenq_e = &aenq->entries[masked_head];
1819                 aenq_common = &aenq_e->aenq_common_desc;
1820         }
1821
1822         aenq->head += processed;
1823         aenq->phase = phase;
1824
1825         /* Don't update aenq doorbell if there weren't any processed events */
1826         if (!processed)
1827                 return;
1828
1829         /* write the aenq doorbell after all AENQ descriptors were read */
1830         mb();
1831         writel_relaxed((u32)aenq->head,
1832                        dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
1833         mmiowb();
1834 }
1835
1836 int ena_com_dev_reset(struct ena_com_dev *ena_dev,
1837                       enum ena_regs_reset_reason_types reset_reason)
1838 {
1839         u32 stat, timeout, cap, reset_val;
1840         int rc;
1841
1842         stat = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
1843         cap = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
1844
1845         if (unlikely((stat == ENA_MMIO_READ_TIMEOUT) ||
1846                      (cap == ENA_MMIO_READ_TIMEOUT))) {
1847                 pr_err("Reg read32 timeout occurred\n");
1848                 return -ETIME;
1849         }
1850
1851         if ((stat & ENA_REGS_DEV_STS_READY_MASK) == 0) {
1852                 pr_err("Device isn't ready, can't reset device\n");
1853                 return -EINVAL;
1854         }
1855
1856         timeout = (cap & ENA_REGS_CAPS_RESET_TIMEOUT_MASK) >>
1857                         ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT;
1858         if (timeout == 0) {
1859                 pr_err("Invalid timeout value\n");
1860                 return -EINVAL;
1861         }
1862
1863         /* start reset */
1864         reset_val = ENA_REGS_DEV_CTL_DEV_RESET_MASK;
1865         reset_val |= (reset_reason << ENA_REGS_DEV_CTL_RESET_REASON_SHIFT) &
1866                      ENA_REGS_DEV_CTL_RESET_REASON_MASK;
1867         writel(reset_val, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
1868
1869         /* Write again the MMIO read request address */
1870         ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
1871
1872         rc = wait_for_reset_state(ena_dev, timeout,
1873                                   ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK);
1874         if (rc != 0) {
1875                 pr_err("Reset indication didn't turn on\n");
1876                 return rc;
1877         }
1878
1879         /* reset done */
1880         writel(0, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
1881         rc = wait_for_reset_state(ena_dev, timeout, 0);
1882         if (rc != 0) {
1883                 pr_err("Reset indication didn't turn off\n");
1884                 return rc;
1885         }
1886
1887         timeout = (cap & ENA_REGS_CAPS_ADMIN_CMD_TO_MASK) >>
1888                 ENA_REGS_CAPS_ADMIN_CMD_TO_SHIFT;
1889         if (timeout)
1890                 /* the resolution of timeout reg is 100ms */
1891                 ena_dev->admin_queue.completion_timeout = timeout * 100000;
1892         else
1893                 ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;
1894
1895         return 0;
1896 }
1897
1898 static int ena_get_dev_stats(struct ena_com_dev *ena_dev,
1899                              struct ena_com_stats_ctx *ctx,
1900                              enum ena_admin_get_stats_type type)
1901 {
1902         struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd;
1903         struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp;
1904         struct ena_com_admin_queue *admin_queue;
1905         int ret;
1906
1907         admin_queue = &ena_dev->admin_queue;
1908
1909         get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS;
1910         get_cmd->aq_common_descriptor.flags = 0;
1911         get_cmd->type = type;
1912
1913         ret =  ena_com_execute_admin_command(admin_queue,
1914                                              (struct ena_admin_aq_entry *)get_cmd,
1915                                              sizeof(*get_cmd),
1916                                              (struct ena_admin_acq_entry *)get_resp,
1917                                              sizeof(*get_resp));
1918
1919         if (unlikely(ret))
1920                 pr_err("Failed to get stats. error: %d\n", ret);
1921
1922         return ret;
1923 }
1924
1925 int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev,
1926                                 struct ena_admin_basic_stats *stats)
1927 {
1928         struct ena_com_stats_ctx ctx;
1929         int ret;
1930
1931         memset(&ctx, 0x0, sizeof(ctx));
1932         ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_BASIC);
1933         if (likely(ret == 0))
1934                 memcpy(stats, &ctx.get_resp.basic_stats,
1935                        sizeof(ctx.get_resp.basic_stats));
1936
1937         return ret;
1938 }
1939
1940 int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu)
1941 {
1942         struct ena_com_admin_queue *admin_queue;
1943         struct ena_admin_set_feat_cmd cmd;
1944         struct ena_admin_set_feat_resp resp;
1945         int ret;
1946
1947         if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) {
1948                 pr_debug("Feature %d isn't supported\n", ENA_ADMIN_MTU);
1949                 return -EOPNOTSUPP;
1950         }
1951
1952         memset(&cmd, 0x0, sizeof(cmd));
1953         admin_queue = &ena_dev->admin_queue;
1954
1955         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
1956         cmd.aq_common_descriptor.flags = 0;
1957         cmd.feat_common.feature_id = ENA_ADMIN_MTU;
1958         cmd.u.mtu.mtu = mtu;
1959
1960         ret = ena_com_execute_admin_command(admin_queue,
1961                                             (struct ena_admin_aq_entry *)&cmd,
1962                                             sizeof(cmd),
1963                                             (struct ena_admin_acq_entry *)&resp,
1964                                             sizeof(resp));
1965
1966         if (unlikely(ret))
1967                 pr_err("Failed to set mtu %d. error: %d\n", mtu, ret);
1968
1969         return ret;
1970 }
1971
1972 int ena_com_get_offload_settings(struct ena_com_dev *ena_dev,
1973                                  struct ena_admin_feature_offload_desc *offload)
1974 {
1975         int ret;
1976         struct ena_admin_get_feat_resp resp;
1977
1978         ret = ena_com_get_feature(ena_dev, &resp,
1979                                   ENA_ADMIN_STATELESS_OFFLOAD_CONFIG);
1980         if (unlikely(ret)) {
1981                 pr_err("Failed to get offload capabilities %d\n", ret);
1982                 return ret;
1983         }
1984
1985         memcpy(offload, &resp.u.offload, sizeof(resp.u.offload));
1986
1987         return 0;
1988 }
1989
1990 int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
1991 {
1992         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1993         struct ena_rss *rss = &ena_dev->rss;
1994         struct ena_admin_set_feat_cmd cmd;
1995         struct ena_admin_set_feat_resp resp;
1996         struct ena_admin_get_feat_resp get_resp;
1997         int ret;
1998
1999         if (!ena_com_check_supported_feature_id(ena_dev,
2000                                                 ENA_ADMIN_RSS_HASH_FUNCTION)) {
2001                 pr_debug("Feature %d isn't supported\n",
2002                          ENA_ADMIN_RSS_HASH_FUNCTION);
2003                 return -EOPNOTSUPP;
2004         }
2005
2006         /* Validate hash function is supported */
2007         ret = ena_com_get_feature(ena_dev, &get_resp,
2008                                   ENA_ADMIN_RSS_HASH_FUNCTION);
2009         if (unlikely(ret))
2010                 return ret;
2011
2012         if (get_resp.u.flow_hash_func.supported_func & (1 << rss->hash_func)) {
2013                 pr_err("Func hash %d isn't supported by device, abort\n",
2014                        rss->hash_func);
2015                 return -EOPNOTSUPP;
2016         }
2017
2018         memset(&cmd, 0x0, sizeof(cmd));
2019
2020         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2021         cmd.aq_common_descriptor.flags =
2022                 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2023         cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_FUNCTION;
2024         cmd.u.flow_hash_func.init_val = rss->hash_init_val;
2025         cmd.u.flow_hash_func.selected_func = 1 << rss->hash_func;
2026
2027         ret = ena_com_mem_addr_set(ena_dev,
2028                                    &cmd.control_buffer.address,
2029                                    rss->hash_key_dma_addr);
2030         if (unlikely(ret)) {
2031                 pr_err("memory address set failed\n");
2032                 return ret;
2033         }
2034
2035         cmd.control_buffer.length = sizeof(*rss->hash_key);
2036
2037         ret = ena_com_execute_admin_command(admin_queue,
2038                                             (struct ena_admin_aq_entry *)&cmd,
2039                                             sizeof(cmd),
2040                                             (struct ena_admin_acq_entry *)&resp,
2041                                             sizeof(resp));
2042         if (unlikely(ret)) {
2043                 pr_err("Failed to set hash function %d. error: %d\n",
2044                        rss->hash_func, ret);
2045                 return -EINVAL;
2046         }
2047
2048         return 0;
2049 }
2050
2051 int ena_com_fill_hash_function(struct ena_com_dev *ena_dev,
2052                                enum ena_admin_hash_functions func,
2053                                const u8 *key, u16 key_len, u32 init_val)
2054 {
2055         struct ena_rss *rss = &ena_dev->rss;
2056         struct ena_admin_get_feat_resp get_resp;
2057         struct ena_admin_feature_rss_flow_hash_control *hash_key =
2058                 rss->hash_key;
2059         int rc;
2060
2061         /* Make sure size is a mult of DWs */
2062         if (unlikely(key_len & 0x3))
2063                 return -EINVAL;
2064
2065         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2066                                     ENA_ADMIN_RSS_HASH_FUNCTION,
2067                                     rss->hash_key_dma_addr,
2068                                     sizeof(*rss->hash_key));
2069         if (unlikely(rc))
2070                 return rc;
2071
2072         if (!((1 << func) & get_resp.u.flow_hash_func.supported_func)) {
2073                 pr_err("Flow hash function %d isn't supported\n", func);
2074                 return -EOPNOTSUPP;
2075         }
2076
2077         switch (func) {
2078         case ENA_ADMIN_TOEPLITZ:
2079                 if (key_len > sizeof(hash_key->key)) {
2080                         pr_err("key len (%hu) is bigger than the max supported (%zu)\n",
2081                                key_len, sizeof(hash_key->key));
2082                         return -EINVAL;
2083                 }
2084
2085                 memcpy(hash_key->key, key, key_len);
2086                 rss->hash_init_val = init_val;
2087                 hash_key->keys_num = key_len >> 2;
2088                 break;
2089         case ENA_ADMIN_CRC32:
2090                 rss->hash_init_val = init_val;
2091                 break;
2092         default:
2093                 pr_err("Invalid hash function (%d)\n", func);
2094                 return -EINVAL;
2095         }
2096
2097         rc = ena_com_set_hash_function(ena_dev);
2098
2099         /* Restore the old function */
2100         if (unlikely(rc))
2101                 ena_com_get_hash_function(ena_dev, NULL, NULL);
2102
2103         return rc;
2104 }
2105
2106 int ena_com_get_hash_function(struct ena_com_dev *ena_dev,
2107                               enum ena_admin_hash_functions *func,
2108                               u8 *key)
2109 {
2110         struct ena_rss *rss = &ena_dev->rss;
2111         struct ena_admin_get_feat_resp get_resp;
2112         struct ena_admin_feature_rss_flow_hash_control *hash_key =
2113                 rss->hash_key;
2114         int rc;
2115
2116         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2117                                     ENA_ADMIN_RSS_HASH_FUNCTION,
2118                                     rss->hash_key_dma_addr,
2119                                     sizeof(*rss->hash_key));
2120         if (unlikely(rc))
2121                 return rc;
2122
2123         rss->hash_func = get_resp.u.flow_hash_func.selected_func;
2124         if (func)
2125                 *func = rss->hash_func;
2126
2127         if (key)
2128                 memcpy(key, hash_key->key, (size_t)(hash_key->keys_num) << 2);
2129
2130         return 0;
2131 }
2132
2133 int ena_com_get_hash_ctrl(struct ena_com_dev *ena_dev,
2134                           enum ena_admin_flow_hash_proto proto,
2135                           u16 *fields)
2136 {
2137         struct ena_rss *rss = &ena_dev->rss;
2138         struct ena_admin_get_feat_resp get_resp;
2139         int rc;
2140
2141         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2142                                     ENA_ADMIN_RSS_HASH_INPUT,
2143                                     rss->hash_ctrl_dma_addr,
2144                                     sizeof(*rss->hash_ctrl));
2145         if (unlikely(rc))
2146                 return rc;
2147
2148         if (fields)
2149                 *fields = rss->hash_ctrl->selected_fields[proto].fields;
2150
2151         return 0;
2152 }
2153
2154 int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
2155 {
2156         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2157         struct ena_rss *rss = &ena_dev->rss;
2158         struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
2159         struct ena_admin_set_feat_cmd cmd;
2160         struct ena_admin_set_feat_resp resp;
2161         int ret;
2162
2163         if (!ena_com_check_supported_feature_id(ena_dev,
2164                                                 ENA_ADMIN_RSS_HASH_INPUT)) {
2165                 pr_debug("Feature %d isn't supported\n",
2166                          ENA_ADMIN_RSS_HASH_INPUT);
2167                 return -EOPNOTSUPP;
2168         }
2169
2170         memset(&cmd, 0x0, sizeof(cmd));
2171
2172         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2173         cmd.aq_common_descriptor.flags =
2174                 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2175         cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_INPUT;
2176         cmd.u.flow_hash_input.enabled_input_sort =
2177                 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK |
2178                 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK;
2179
2180         ret = ena_com_mem_addr_set(ena_dev,
2181                                    &cmd.control_buffer.address,
2182                                    rss->hash_ctrl_dma_addr);
2183         if (unlikely(ret)) {
2184                 pr_err("memory address set failed\n");
2185                 return ret;
2186         }
2187         cmd.control_buffer.length = sizeof(*hash_ctrl);
2188
2189         ret = ena_com_execute_admin_command(admin_queue,
2190                                             (struct ena_admin_aq_entry *)&cmd,
2191                                             sizeof(cmd),
2192                                             (struct ena_admin_acq_entry *)&resp,
2193                                             sizeof(resp));
2194         if (unlikely(ret))
2195                 pr_err("Failed to set hash input. error: %d\n", ret);
2196
2197         return ret;
2198 }
2199
2200 int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
2201 {
2202         struct ena_rss *rss = &ena_dev->rss;
2203         struct ena_admin_feature_rss_hash_control *hash_ctrl =
2204                 rss->hash_ctrl;
2205         u16 available_fields = 0;
2206         int rc, i;
2207
2208         /* Get the supported hash input */
2209         rc = ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2210         if (unlikely(rc))
2211                 return rc;
2212
2213         hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP4].fields =
2214                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2215                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2216
2217         hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP4].fields =
2218                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2219                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2220
2221         hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP6].fields =
2222                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2223                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2224
2225         hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP6].fields =
2226                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2227                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2228
2229         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4].fields =
2230                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2231
2232         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP6].fields =
2233                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2234
2235         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
2236                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2237
2238         hash_ctrl->selected_fields[ENA_ADMIN_RSS_NOT_IP].fields =
2239                 ENA_ADMIN_RSS_L2_DA | ENA_ADMIN_RSS_L2_SA;
2240
2241         for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) {
2242                 available_fields = hash_ctrl->selected_fields[i].fields &
2243                                 hash_ctrl->supported_fields[i].fields;
2244                 if (available_fields != hash_ctrl->selected_fields[i].fields) {
2245                         pr_err("hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n",
2246                                i, hash_ctrl->supported_fields[i].fields,
2247                                hash_ctrl->selected_fields[i].fields);
2248                         return -EOPNOTSUPP;
2249                 }
2250         }
2251
2252         rc = ena_com_set_hash_ctrl(ena_dev);
2253
2254         /* In case of failure, restore the old hash ctrl */
2255         if (unlikely(rc))
2256                 ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2257
2258         return rc;
2259 }
2260
2261 int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev,
2262                            enum ena_admin_flow_hash_proto proto,
2263                            u16 hash_fields)
2264 {
2265         struct ena_rss *rss = &ena_dev->rss;
2266         struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
2267         u16 supported_fields;
2268         int rc;
2269
2270         if (proto >= ENA_ADMIN_RSS_PROTO_NUM) {
2271                 pr_err("Invalid proto num (%u)\n", proto);
2272                 return -EINVAL;
2273         }
2274
2275         /* Get the ctrl table */
2276         rc = ena_com_get_hash_ctrl(ena_dev, proto, NULL);
2277         if (unlikely(rc))
2278                 return rc;
2279
2280         /* Make sure all the fields are supported */
2281         supported_fields = hash_ctrl->supported_fields[proto].fields;
2282         if ((hash_fields & supported_fields) != hash_fields) {
2283                 pr_err("proto %d doesn't support the required fields %x. supports only: %x\n",
2284                        proto, hash_fields, supported_fields);
2285         }
2286
2287         hash_ctrl->selected_fields[proto].fields = hash_fields;
2288
2289         rc = ena_com_set_hash_ctrl(ena_dev);
2290
2291         /* In case of failure, restore the old hash ctrl */
2292         if (unlikely(rc))
2293                 ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2294
2295         return 0;
2296 }
2297
2298 int ena_com_indirect_table_fill_entry(struct ena_com_dev *ena_dev,
2299                                       u16 entry_idx, u16 entry_value)
2300 {
2301         struct ena_rss *rss = &ena_dev->rss;
2302
2303         if (unlikely(entry_idx >= (1 << rss->tbl_log_size)))
2304                 return -EINVAL;
2305
2306         if (unlikely((entry_value > ENA_TOTAL_NUM_QUEUES)))
2307                 return -EINVAL;
2308
2309         rss->host_rss_ind_tbl[entry_idx] = entry_value;
2310
2311         return 0;
2312 }
2313
2314 int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
2315 {
2316         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2317         struct ena_rss *rss = &ena_dev->rss;
2318         struct ena_admin_set_feat_cmd cmd;
2319         struct ena_admin_set_feat_resp resp;
2320         int ret;
2321
2322         if (!ena_com_check_supported_feature_id(
2323                     ena_dev, ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG)) {
2324                 pr_debug("Feature %d isn't supported\n",
2325                          ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
2326                 return -EOPNOTSUPP;
2327         }
2328
2329         ret = ena_com_ind_tbl_convert_to_device(ena_dev);
2330         if (ret) {
2331                 pr_err("Failed to convert host indirection table to device table\n");
2332                 return ret;
2333         }
2334
2335         memset(&cmd, 0x0, sizeof(cmd));
2336
2337         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2338         cmd.aq_common_descriptor.flags =
2339                 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2340         cmd.feat_common.feature_id = ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG;
2341         cmd.u.ind_table.size = rss->tbl_log_size;
2342         cmd.u.ind_table.inline_index = 0xFFFFFFFF;
2343
2344         ret = ena_com_mem_addr_set(ena_dev,
2345                                    &cmd.control_buffer.address,
2346                                    rss->rss_ind_tbl_dma_addr);
2347         if (unlikely(ret)) {
2348                 pr_err("memory address set failed\n");
2349                 return ret;
2350         }
2351
2352         cmd.control_buffer.length = (1ULL << rss->tbl_log_size) *
2353                 sizeof(struct ena_admin_rss_ind_table_entry);
2354
2355         ret = ena_com_execute_admin_command(admin_queue,
2356                                             (struct ena_admin_aq_entry *)&cmd,
2357                                             sizeof(cmd),
2358                                             (struct ena_admin_acq_entry *)&resp,
2359                                             sizeof(resp));
2360
2361         if (unlikely(ret))
2362                 pr_err("Failed to set indirect table. error: %d\n", ret);
2363
2364         return ret;
2365 }
2366
2367 int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl)
2368 {
2369         struct ena_rss *rss = &ena_dev->rss;
2370         struct ena_admin_get_feat_resp get_resp;
2371         u32 tbl_size;
2372         int i, rc;
2373
2374         tbl_size = (1ULL << rss->tbl_log_size) *
2375                 sizeof(struct ena_admin_rss_ind_table_entry);
2376
2377         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2378                                     ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG,
2379                                     rss->rss_ind_tbl_dma_addr,
2380                                     tbl_size);
2381         if (unlikely(rc))
2382                 return rc;
2383
2384         if (!ind_tbl)
2385                 return 0;
2386
2387         rc = ena_com_ind_tbl_convert_from_device(ena_dev);
2388         if (unlikely(rc))
2389                 return rc;
2390
2391         for (i = 0; i < (1 << rss->tbl_log_size); i++)
2392                 ind_tbl[i] = rss->host_rss_ind_tbl[i];
2393
2394         return 0;
2395 }
2396
2397 int ena_com_rss_init(struct ena_com_dev *ena_dev, u16 indr_tbl_log_size)
2398 {
2399         int rc;
2400
2401         memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
2402
2403         rc = ena_com_indirect_table_allocate(ena_dev, indr_tbl_log_size);
2404         if (unlikely(rc))
2405                 goto err_indr_tbl;
2406
2407         rc = ena_com_hash_key_allocate(ena_dev);
2408         if (unlikely(rc))
2409                 goto err_hash_key;
2410
2411         rc = ena_com_hash_ctrl_init(ena_dev);
2412         if (unlikely(rc))
2413                 goto err_hash_ctrl;
2414
2415         return 0;
2416
2417 err_hash_ctrl:
2418         ena_com_hash_key_destroy(ena_dev);
2419 err_hash_key:
2420         ena_com_indirect_table_destroy(ena_dev);
2421 err_indr_tbl:
2422
2423         return rc;
2424 }
2425
2426 void ena_com_rss_destroy(struct ena_com_dev *ena_dev)
2427 {
2428         ena_com_indirect_table_destroy(ena_dev);
2429         ena_com_hash_key_destroy(ena_dev);
2430         ena_com_hash_ctrl_destroy(ena_dev);
2431
2432         memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
2433 }
2434
2435 int ena_com_allocate_host_info(struct ena_com_dev *ena_dev)
2436 {
2437         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2438
2439         host_attr->host_info =
2440                 dma_zalloc_coherent(ena_dev->dmadev, SZ_4K,
2441                                     &host_attr->host_info_dma_addr, GFP_KERNEL);
2442         if (unlikely(!host_attr->host_info))
2443                 return -ENOMEM;
2444
2445         return 0;
2446 }
2447
2448 int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev,
2449                                 u32 debug_area_size)
2450 {
2451         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2452
2453         host_attr->debug_area_virt_addr =
2454                 dma_zalloc_coherent(ena_dev->dmadev, debug_area_size,
2455                                     &host_attr->debug_area_dma_addr, GFP_KERNEL);
2456         if (unlikely(!host_attr->debug_area_virt_addr)) {
2457                 host_attr->debug_area_size = 0;
2458                 return -ENOMEM;
2459         }
2460
2461         host_attr->debug_area_size = debug_area_size;
2462
2463         return 0;
2464 }
2465
2466 void ena_com_delete_host_info(struct ena_com_dev *ena_dev)
2467 {
2468         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2469
2470         if (host_attr->host_info) {
2471                 dma_free_coherent(ena_dev->dmadev, SZ_4K, host_attr->host_info,
2472                                   host_attr->host_info_dma_addr);
2473                 host_attr->host_info = NULL;
2474         }
2475 }
2476
2477 void ena_com_delete_debug_area(struct ena_com_dev *ena_dev)
2478 {
2479         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2480
2481         if (host_attr->debug_area_virt_addr) {
2482                 dma_free_coherent(ena_dev->dmadev, host_attr->debug_area_size,
2483                                   host_attr->debug_area_virt_addr,
2484                                   host_attr->debug_area_dma_addr);
2485                 host_attr->debug_area_virt_addr = NULL;
2486         }
2487 }
2488
2489 int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
2490 {
2491         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2492         struct ena_com_admin_queue *admin_queue;
2493         struct ena_admin_set_feat_cmd cmd;
2494         struct ena_admin_set_feat_resp resp;
2495
2496         int ret;
2497
2498         /* Host attribute config is called before ena_com_get_dev_attr_feat
2499          * so ena_com can't check if the feature is supported.
2500          */
2501
2502         memset(&cmd, 0x0, sizeof(cmd));
2503         admin_queue = &ena_dev->admin_queue;
2504
2505         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2506         cmd.feat_common.feature_id = ENA_ADMIN_HOST_ATTR_CONFIG;
2507
2508         ret = ena_com_mem_addr_set(ena_dev,
2509                                    &cmd.u.host_attr.debug_ba,
2510                                    host_attr->debug_area_dma_addr);
2511         if (unlikely(ret)) {
2512                 pr_err("memory address set failed\n");
2513                 return ret;
2514         }
2515
2516         ret = ena_com_mem_addr_set(ena_dev,
2517                                    &cmd.u.host_attr.os_info_ba,
2518                                    host_attr->host_info_dma_addr);
2519         if (unlikely(ret)) {
2520                 pr_err("memory address set failed\n");
2521                 return ret;
2522         }
2523
2524         cmd.u.host_attr.debug_area_size = host_attr->debug_area_size;
2525
2526         ret = ena_com_execute_admin_command(admin_queue,
2527                                             (struct ena_admin_aq_entry *)&cmd,
2528                                             sizeof(cmd),
2529                                             (struct ena_admin_acq_entry *)&resp,
2530                                             sizeof(resp));
2531
2532         if (unlikely(ret))
2533                 pr_err("Failed to set host attributes: %d\n", ret);
2534
2535         return ret;
2536 }
2537
2538 /* Interrupt moderation */
2539 bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev)
2540 {
2541         return ena_com_check_supported_feature_id(ena_dev,
2542                                                   ENA_ADMIN_INTERRUPT_MODERATION);
2543 }
2544
2545 int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
2546                                                       u32 tx_coalesce_usecs)
2547 {
2548         if (!ena_dev->intr_delay_resolution) {
2549                 pr_err("Illegal interrupt delay granularity value\n");
2550                 return -EFAULT;
2551         }
2552
2553         ena_dev->intr_moder_tx_interval = tx_coalesce_usecs /
2554                 ena_dev->intr_delay_resolution;
2555
2556         return 0;
2557 }
2558
2559 int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
2560                                                       u32 rx_coalesce_usecs)
2561 {
2562         if (!ena_dev->intr_delay_resolution) {
2563                 pr_err("Illegal interrupt delay granularity value\n");
2564                 return -EFAULT;
2565         }
2566
2567         /* We use LOWEST entry of moderation table for storing
2568          * nonadaptive interrupt coalescing values
2569          */
2570         ena_dev->intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
2571                 rx_coalesce_usecs / ena_dev->intr_delay_resolution;
2572
2573         return 0;
2574 }
2575
2576 void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev)
2577 {
2578         if (ena_dev->intr_moder_tbl)
2579                 devm_kfree(ena_dev->dmadev, ena_dev->intr_moder_tbl);
2580         ena_dev->intr_moder_tbl = NULL;
2581 }
2582
2583 int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev)
2584 {
2585         struct ena_admin_get_feat_resp get_resp;
2586         u16 delay_resolution;
2587         int rc;
2588
2589         rc = ena_com_get_feature(ena_dev, &get_resp,
2590                                  ENA_ADMIN_INTERRUPT_MODERATION);
2591
2592         if (rc) {
2593                 if (rc == -EOPNOTSUPP) {
2594                         pr_debug("Feature %d isn't supported\n",
2595                                  ENA_ADMIN_INTERRUPT_MODERATION);
2596                         rc = 0;
2597                 } else {
2598                         pr_err("Failed to get interrupt moderation admin cmd. rc: %d\n",
2599                                rc);
2600                 }
2601
2602                 /* no moderation supported, disable adaptive support */
2603                 ena_com_disable_adaptive_moderation(ena_dev);
2604                 return rc;
2605         }
2606
2607         rc = ena_com_init_interrupt_moderation_table(ena_dev);
2608         if (rc)
2609                 goto err;
2610
2611         /* if moderation is supported by device we set adaptive moderation */
2612         delay_resolution = get_resp.u.intr_moderation.intr_delay_resolution;
2613         ena_com_update_intr_delay_resolution(ena_dev, delay_resolution);
2614         ena_com_enable_adaptive_moderation(ena_dev);
2615
2616         return 0;
2617 err:
2618         ena_com_destroy_interrupt_moderation(ena_dev);
2619         return rc;
2620 }
2621
2622 void ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev)
2623 {
2624         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2625
2626         if (!intr_moder_tbl)
2627                 return;
2628
2629         intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
2630                 ENA_INTR_LOWEST_USECS;
2631         intr_moder_tbl[ENA_INTR_MODER_LOWEST].pkts_per_interval =
2632                 ENA_INTR_LOWEST_PKTS;
2633         intr_moder_tbl[ENA_INTR_MODER_LOWEST].bytes_per_interval =
2634                 ENA_INTR_LOWEST_BYTES;
2635
2636         intr_moder_tbl[ENA_INTR_MODER_LOW].intr_moder_interval =
2637                 ENA_INTR_LOW_USECS;
2638         intr_moder_tbl[ENA_INTR_MODER_LOW].pkts_per_interval =
2639                 ENA_INTR_LOW_PKTS;
2640         intr_moder_tbl[ENA_INTR_MODER_LOW].bytes_per_interval =
2641                 ENA_INTR_LOW_BYTES;
2642
2643         intr_moder_tbl[ENA_INTR_MODER_MID].intr_moder_interval =
2644                 ENA_INTR_MID_USECS;
2645         intr_moder_tbl[ENA_INTR_MODER_MID].pkts_per_interval =
2646                 ENA_INTR_MID_PKTS;
2647         intr_moder_tbl[ENA_INTR_MODER_MID].bytes_per_interval =
2648                 ENA_INTR_MID_BYTES;
2649
2650         intr_moder_tbl[ENA_INTR_MODER_HIGH].intr_moder_interval =
2651                 ENA_INTR_HIGH_USECS;
2652         intr_moder_tbl[ENA_INTR_MODER_HIGH].pkts_per_interval =
2653                 ENA_INTR_HIGH_PKTS;
2654         intr_moder_tbl[ENA_INTR_MODER_HIGH].bytes_per_interval =
2655                 ENA_INTR_HIGH_BYTES;
2656
2657         intr_moder_tbl[ENA_INTR_MODER_HIGHEST].intr_moder_interval =
2658                 ENA_INTR_HIGHEST_USECS;
2659         intr_moder_tbl[ENA_INTR_MODER_HIGHEST].pkts_per_interval =
2660                 ENA_INTR_HIGHEST_PKTS;
2661         intr_moder_tbl[ENA_INTR_MODER_HIGHEST].bytes_per_interval =
2662                 ENA_INTR_HIGHEST_BYTES;
2663 }
2664
2665 unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev)
2666 {
2667         return ena_dev->intr_moder_tx_interval;
2668 }
2669
2670 unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev)
2671 {
2672         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2673
2674         if (intr_moder_tbl)
2675                 return intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval;
2676
2677         return 0;
2678 }
2679
2680 void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev,
2681                                         enum ena_intr_moder_level level,
2682                                         struct ena_intr_moder_entry *entry)
2683 {
2684         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2685
2686         if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
2687                 return;
2688
2689         intr_moder_tbl[level].intr_moder_interval = entry->intr_moder_interval;
2690         if (ena_dev->intr_delay_resolution)
2691                 intr_moder_tbl[level].intr_moder_interval /=
2692                         ena_dev->intr_delay_resolution;
2693         intr_moder_tbl[level].pkts_per_interval = entry->pkts_per_interval;
2694
2695         /* use hardcoded value until ethtool supports bytecount parameter */
2696         if (entry->bytes_per_interval != ENA_INTR_BYTE_COUNT_NOT_SUPPORTED)
2697                 intr_moder_tbl[level].bytes_per_interval = entry->bytes_per_interval;
2698 }
2699
2700 void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
2701                                        enum ena_intr_moder_level level,
2702                                        struct ena_intr_moder_entry *entry)
2703 {
2704         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2705
2706         if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
2707                 return;
2708
2709         entry->intr_moder_interval = intr_moder_tbl[level].intr_moder_interval;
2710         if (ena_dev->intr_delay_resolution)
2711                 entry->intr_moder_interval *= ena_dev->intr_delay_resolution;
2712         entry->pkts_per_interval =
2713         intr_moder_tbl[level].pkts_per_interval;
2714         entry->bytes_per_interval = intr_moder_tbl[level].bytes_per_interval;
2715 }