2129fbd1d6cab96b95ba5b9af65d5cebdb85cefc
[sfrench/cifs-2.6.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_main.c
1 /*
2  * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32
33 #include <net/tc_act/tc_gact.h>
34 #include <net/pkt_cls.h>
35 #include <linux/mlx5/fs.h>
36 #include <net/vxlan.h>
37 #include <linux/bpf.h>
38 #include "en.h"
39 #include "en_tc.h"
40 #include "eswitch.h"
41 #include "vxlan.h"
42
43 struct mlx5e_rq_param {
44         u32                     rqc[MLX5_ST_SZ_DW(rqc)];
45         struct mlx5_wq_param    wq;
46         bool                    am_enabled;
47 };
48
49 struct mlx5e_sq_param {
50         u32                        sqc[MLX5_ST_SZ_DW(sqc)];
51         struct mlx5_wq_param       wq;
52         u16                        max_inline;
53         u8                         min_inline_mode;
54         enum mlx5e_sq_type         type;
55 };
56
57 struct mlx5e_cq_param {
58         u32                        cqc[MLX5_ST_SZ_DW(cqc)];
59         struct mlx5_wq_param       wq;
60         u16                        eq_ix;
61         u8                         cq_period_mode;
62 };
63
64 struct mlx5e_channel_param {
65         struct mlx5e_rq_param      rq;
66         struct mlx5e_sq_param      sq;
67         struct mlx5e_sq_param      xdp_sq;
68         struct mlx5e_sq_param      icosq;
69         struct mlx5e_cq_param      rx_cq;
70         struct mlx5e_cq_param      tx_cq;
71         struct mlx5e_cq_param      icosq_cq;
72 };
73
74 static bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
75 {
76         return MLX5_CAP_GEN(mdev, striding_rq) &&
77                 MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
78                 MLX5_CAP_ETH(mdev, reg_umr_sq);
79 }
80
81 static void mlx5e_set_rq_type_params(struct mlx5e_priv *priv, u8 rq_type)
82 {
83         priv->params.rq_wq_type = rq_type;
84         switch (priv->params.rq_wq_type) {
85         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
86                 priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW;
87                 priv->params.mpwqe_log_stride_sz =
88                         MLX5E_GET_PFLAG(priv, MLX5E_PFLAG_RX_CQE_COMPRESS) ?
89                         MLX5_MPWRQ_LOG_STRIDE_SIZE_CQE_COMPRESS :
90                         MLX5_MPWRQ_LOG_STRIDE_SIZE;
91                 priv->params.mpwqe_log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ -
92                         priv->params.mpwqe_log_stride_sz;
93                 break;
94         default: /* MLX5_WQ_TYPE_LINKED_LIST */
95                 priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
96         }
97         priv->params.min_rx_wqes = mlx5_min_rx_wqes(priv->params.rq_wq_type,
98                                                BIT(priv->params.log_rq_size));
99
100         mlx5_core_info(priv->mdev,
101                        "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
102                        priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
103                        BIT(priv->params.log_rq_size),
104                        BIT(priv->params.mpwqe_log_stride_sz),
105                        MLX5E_GET_PFLAG(priv, MLX5E_PFLAG_RX_CQE_COMPRESS));
106 }
107
108 static void mlx5e_set_rq_priv_params(struct mlx5e_priv *priv)
109 {
110         u8 rq_type = mlx5e_check_fragmented_striding_rq_cap(priv->mdev) &&
111                     !priv->xdp_prog ?
112                     MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
113                     MLX5_WQ_TYPE_LINKED_LIST;
114         mlx5e_set_rq_type_params(priv, rq_type);
115 }
116
117 static void mlx5e_update_carrier(struct mlx5e_priv *priv)
118 {
119         struct mlx5_core_dev *mdev = priv->mdev;
120         u8 port_state;
121
122         port_state = mlx5_query_vport_state(mdev,
123                 MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
124
125         if (port_state == VPORT_STATE_UP) {
126                 netdev_info(priv->netdev, "Link up\n");
127                 netif_carrier_on(priv->netdev);
128         } else {
129                 netdev_info(priv->netdev, "Link down\n");
130                 netif_carrier_off(priv->netdev);
131         }
132 }
133
134 static void mlx5e_update_carrier_work(struct work_struct *work)
135 {
136         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
137                                                update_carrier_work);
138
139         mutex_lock(&priv->state_lock);
140         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
141                 mlx5e_update_carrier(priv);
142         mutex_unlock(&priv->state_lock);
143 }
144
145 static void mlx5e_tx_timeout_work(struct work_struct *work)
146 {
147         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
148                                                tx_timeout_work);
149         int err;
150
151         rtnl_lock();
152         mutex_lock(&priv->state_lock);
153         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
154                 goto unlock;
155         mlx5e_close_locked(priv->netdev);
156         err = mlx5e_open_locked(priv->netdev);
157         if (err)
158                 netdev_err(priv->netdev, "mlx5e_open_locked failed recovering from a tx_timeout, err(%d).\n",
159                            err);
160 unlock:
161         mutex_unlock(&priv->state_lock);
162         rtnl_unlock();
163 }
164
165 static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
166 {
167         struct mlx5e_sw_stats *s = &priv->stats.sw;
168         struct mlx5e_rq_stats *rq_stats;
169         struct mlx5e_sq_stats *sq_stats;
170         u64 tx_offload_none = 0;
171         int i, j;
172
173         memset(s, 0, sizeof(*s));
174         for (i = 0; i < priv->params.num_channels; i++) {
175                 rq_stats = &priv->channel[i]->rq.stats;
176
177                 s->rx_packets   += rq_stats->packets;
178                 s->rx_bytes     += rq_stats->bytes;
179                 s->rx_lro_packets += rq_stats->lro_packets;
180                 s->rx_lro_bytes += rq_stats->lro_bytes;
181                 s->rx_csum_none += rq_stats->csum_none;
182                 s->rx_csum_complete += rq_stats->csum_complete;
183                 s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
184                 s->rx_xdp_drop += rq_stats->xdp_drop;
185                 s->rx_xdp_tx += rq_stats->xdp_tx;
186                 s->rx_xdp_tx_full += rq_stats->xdp_tx_full;
187                 s->rx_wqe_err   += rq_stats->wqe_err;
188                 s->rx_mpwqe_filler += rq_stats->mpwqe_filler;
189                 s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
190                 s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
191                 s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
192                 s->rx_cache_reuse += rq_stats->cache_reuse;
193                 s->rx_cache_full  += rq_stats->cache_full;
194                 s->rx_cache_empty += rq_stats->cache_empty;
195                 s->rx_cache_busy  += rq_stats->cache_busy;
196
197                 for (j = 0; j < priv->params.num_tc; j++) {
198                         sq_stats = &priv->channel[i]->sq[j].stats;
199
200                         s->tx_packets           += sq_stats->packets;
201                         s->tx_bytes             += sq_stats->bytes;
202                         s->tx_tso_packets       += sq_stats->tso_packets;
203                         s->tx_tso_bytes         += sq_stats->tso_bytes;
204                         s->tx_tso_inner_packets += sq_stats->tso_inner_packets;
205                         s->tx_tso_inner_bytes   += sq_stats->tso_inner_bytes;
206                         s->tx_queue_stopped     += sq_stats->stopped;
207                         s->tx_queue_wake        += sq_stats->wake;
208                         s->tx_queue_dropped     += sq_stats->dropped;
209                         s->tx_xmit_more         += sq_stats->xmit_more;
210                         s->tx_csum_partial_inner += sq_stats->csum_partial_inner;
211                         tx_offload_none         += sq_stats->csum_none;
212                 }
213         }
214
215         /* Update calculated offload counters */
216         s->tx_csum_partial = s->tx_packets - tx_offload_none - s->tx_csum_partial_inner;
217         s->rx_csum_unnecessary = s->rx_packets - s->rx_csum_none - s->rx_csum_complete;
218
219         s->link_down_events_phy = MLX5_GET(ppcnt_reg,
220                                 priv->stats.pport.phy_counters,
221                                 counter_set.phys_layer_cntrs.link_down_events);
222 }
223
224 static void mlx5e_update_vport_counters(struct mlx5e_priv *priv)
225 {
226         int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
227         u32 *out = (u32 *)priv->stats.vport.query_vport_out;
228         u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
229         struct mlx5_core_dev *mdev = priv->mdev;
230
231         MLX5_SET(query_vport_counter_in, in, opcode,
232                  MLX5_CMD_OP_QUERY_VPORT_COUNTER);
233         MLX5_SET(query_vport_counter_in, in, op_mod, 0);
234         MLX5_SET(query_vport_counter_in, in, other_vport, 0);
235
236         memset(out, 0, outlen);
237         mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
238 }
239
240 static void mlx5e_update_pport_counters(struct mlx5e_priv *priv)
241 {
242         struct mlx5e_pport_stats *pstats = &priv->stats.pport;
243         struct mlx5_core_dev *mdev = priv->mdev;
244         int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
245         int prio;
246         void *out;
247         u32 *in;
248
249         in = mlx5_vzalloc(sz);
250         if (!in)
251                 goto free_out;
252
253         MLX5_SET(ppcnt_reg, in, local_port, 1);
254
255         out = pstats->IEEE_802_3_counters;
256         MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
257         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
258
259         out = pstats->RFC_2863_counters;
260         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
261         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
262
263         out = pstats->RFC_2819_counters;
264         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
265         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
266
267         out = pstats->phy_counters;
268         MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
269         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
270
271         MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
272         for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
273                 out = pstats->per_prio_counters[prio];
274                 MLX5_SET(ppcnt_reg, in, prio_tc, prio);
275                 mlx5_core_access_reg(mdev, in, sz, out, sz,
276                                      MLX5_REG_PPCNT, 0, 0);
277         }
278
279 free_out:
280         kvfree(in);
281 }
282
283 static void mlx5e_update_q_counter(struct mlx5e_priv *priv)
284 {
285         struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt;
286
287         if (!priv->q_counter)
288                 return;
289
290         mlx5_core_query_out_of_buffer(priv->mdev, priv->q_counter,
291                                       &qcnt->rx_out_of_buffer);
292 }
293
294 void mlx5e_update_stats(struct mlx5e_priv *priv)
295 {
296         mlx5e_update_q_counter(priv);
297         mlx5e_update_vport_counters(priv);
298         mlx5e_update_pport_counters(priv);
299         mlx5e_update_sw_counters(priv);
300 }
301
302 void mlx5e_update_stats_work(struct work_struct *work)
303 {
304         struct delayed_work *dwork = to_delayed_work(work);
305         struct mlx5e_priv *priv = container_of(dwork, struct mlx5e_priv,
306                                                update_stats_work);
307         mutex_lock(&priv->state_lock);
308         if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
309                 priv->profile->update_stats(priv);
310                 queue_delayed_work(priv->wq, dwork,
311                                    msecs_to_jiffies(MLX5E_UPDATE_STATS_INTERVAL));
312         }
313         mutex_unlock(&priv->state_lock);
314 }
315
316 static void mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
317                               enum mlx5_dev_event event, unsigned long param)
318 {
319         struct mlx5e_priv *priv = vpriv;
320         struct ptp_clock_event ptp_event;
321         struct mlx5_eqe *eqe = NULL;
322
323         if (!test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state))
324                 return;
325
326         switch (event) {
327         case MLX5_DEV_EVENT_PORT_UP:
328         case MLX5_DEV_EVENT_PORT_DOWN:
329                 queue_work(priv->wq, &priv->update_carrier_work);
330                 break;
331         case MLX5_DEV_EVENT_PPS:
332                 eqe = (struct mlx5_eqe *)param;
333                 ptp_event.type = PTP_CLOCK_EXTTS;
334                 ptp_event.index = eqe->data.pps.pin;
335                 ptp_event.timestamp =
336                         timecounter_cyc2time(&priv->tstamp.clock,
337                                              be64_to_cpu(eqe->data.pps.time_stamp));
338                 mlx5e_pps_event_handler(vpriv, &ptp_event);
339                 break;
340         default:
341                 break;
342         }
343 }
344
345 static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
346 {
347         set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
348 }
349
350 static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
351 {
352         clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
353         synchronize_irq(mlx5_get_msix_vec(priv->mdev, MLX5_EQ_VEC_ASYNC));
354 }
355
356 static inline int mlx5e_get_wqe_mtt_sz(void)
357 {
358         /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
359          * To avoid copying garbage after the mtt array, we allocate
360          * a little more.
361          */
362         return ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(__be64),
363                      MLX5_UMR_MTT_ALIGNMENT);
364 }
365
366 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, struct mlx5e_sq *sq,
367                                        struct mlx5e_umr_wqe *wqe, u16 ix)
368 {
369         struct mlx5_wqe_ctrl_seg      *cseg = &wqe->ctrl;
370         struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
371         struct mlx5_wqe_data_seg      *dseg = &wqe->data;
372         struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
373         u8 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
374         u32 umr_wqe_mtt_offset = mlx5e_get_wqe_mtt_offset(rq, ix);
375
376         cseg->qpn_ds    = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
377                                       ds_cnt);
378         cseg->fm_ce_se  = MLX5_WQE_CTRL_CQ_UPDATE;
379         cseg->imm       = rq->mkey_be;
380
381         ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN;
382         ucseg->xlt_octowords =
383                 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
384         ucseg->bsf_octowords =
385                 cpu_to_be16(MLX5_MTT_OCTW(umr_wqe_mtt_offset));
386         ucseg->mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
387
388         dseg->lkey = sq->mkey_be;
389         dseg->addr = cpu_to_be64(wi->umr.mtt_addr);
390 }
391
392 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq,
393                                      struct mlx5e_channel *c)
394 {
395         int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
396         int mtt_sz = mlx5e_get_wqe_mtt_sz();
397         int mtt_alloc = mtt_sz + MLX5_UMR_ALIGN - 1;
398         int i;
399
400         rq->mpwqe.info = kzalloc_node(wq_sz * sizeof(*rq->mpwqe.info),
401                                       GFP_KERNEL, cpu_to_node(c->cpu));
402         if (!rq->mpwqe.info)
403                 goto err_out;
404
405         /* We allocate more than mtt_sz as we will align the pointer */
406         rq->mpwqe.mtt_no_align = kzalloc_node(mtt_alloc * wq_sz, GFP_KERNEL,
407                                         cpu_to_node(c->cpu));
408         if (unlikely(!rq->mpwqe.mtt_no_align))
409                 goto err_free_wqe_info;
410
411         for (i = 0; i < wq_sz; i++) {
412                 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
413
414                 wi->umr.mtt = PTR_ALIGN(rq->mpwqe.mtt_no_align + i * mtt_alloc,
415                                         MLX5_UMR_ALIGN);
416                 wi->umr.mtt_addr = dma_map_single(c->pdev, wi->umr.mtt, mtt_sz,
417                                                   PCI_DMA_TODEVICE);
418                 if (unlikely(dma_mapping_error(c->pdev, wi->umr.mtt_addr)))
419                         goto err_unmap_mtts;
420
421                 mlx5e_build_umr_wqe(rq, &c->icosq, &wi->umr.wqe, i);
422         }
423
424         return 0;
425
426 err_unmap_mtts:
427         while (--i >= 0) {
428                 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
429
430                 dma_unmap_single(c->pdev, wi->umr.mtt_addr, mtt_sz,
431                                  PCI_DMA_TODEVICE);
432         }
433         kfree(rq->mpwqe.mtt_no_align);
434 err_free_wqe_info:
435         kfree(rq->mpwqe.info);
436
437 err_out:
438         return -ENOMEM;
439 }
440
441 static void mlx5e_rq_free_mpwqe_info(struct mlx5e_rq *rq)
442 {
443         int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
444         int mtt_sz = mlx5e_get_wqe_mtt_sz();
445         int i;
446
447         for (i = 0; i < wq_sz; i++) {
448                 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
449
450                 dma_unmap_single(rq->pdev, wi->umr.mtt_addr, mtt_sz,
451                                  PCI_DMA_TODEVICE);
452         }
453         kfree(rq->mpwqe.mtt_no_align);
454         kfree(rq->mpwqe.info);
455 }
456
457 static int mlx5e_create_umr_mkey(struct mlx5e_priv *priv,
458                                  u64 npages, u8 page_shift,
459                                  struct mlx5_core_mkey *umr_mkey)
460 {
461         struct mlx5_core_dev *mdev = priv->mdev;
462         int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
463         void *mkc;
464         u32 *in;
465         int err;
466
467         if (!MLX5E_VALID_NUM_MTTS(npages))
468                 return -EINVAL;
469
470         in = mlx5_vzalloc(inlen);
471         if (!in)
472                 return -ENOMEM;
473
474         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
475
476         MLX5_SET(mkc, mkc, free, 1);
477         MLX5_SET(mkc, mkc, umr_en, 1);
478         MLX5_SET(mkc, mkc, lw, 1);
479         MLX5_SET(mkc, mkc, lr, 1);
480         MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT);
481
482         MLX5_SET(mkc, mkc, qpn, 0xffffff);
483         MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn);
484         MLX5_SET64(mkc, mkc, len, npages << page_shift);
485         MLX5_SET(mkc, mkc, translations_octword_size,
486                  MLX5_MTT_OCTW(npages));
487         MLX5_SET(mkc, mkc, log_page_size, page_shift);
488
489         err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
490
491         kvfree(in);
492         return err;
493 }
494
495 static int mlx5e_create_rq_umr_mkey(struct mlx5e_rq *rq)
496 {
497         struct mlx5e_priv *priv = rq->priv;
498         u64 num_mtts = MLX5E_REQUIRED_MTTS(BIT(priv->params.log_rq_size));
499
500         return mlx5e_create_umr_mkey(priv, num_mtts, PAGE_SHIFT, &rq->umr_mkey);
501 }
502
503 static int mlx5e_create_rq(struct mlx5e_channel *c,
504                            struct mlx5e_rq_param *param,
505                            struct mlx5e_rq *rq)
506 {
507         struct mlx5e_priv *priv = c->priv;
508         struct mlx5_core_dev *mdev = priv->mdev;
509         void *rqc = param->rqc;
510         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
511         u32 byte_count;
512         u32 frag_sz;
513         int npages;
514         int wq_sz;
515         int err;
516         int i;
517
518         param->wq.db_numa_node = cpu_to_node(c->cpu);
519
520         err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
521                                 &rq->wq_ctrl);
522         if (err)
523                 return err;
524
525         rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
526
527         wq_sz = mlx5_wq_ll_get_size(&rq->wq);
528
529         rq->wq_type = priv->params.rq_wq_type;
530         rq->pdev    = c->pdev;
531         rq->netdev  = c->netdev;
532         rq->tstamp  = &priv->tstamp;
533         rq->channel = c;
534         rq->ix      = c->ix;
535         rq->priv    = c->priv;
536
537         rq->xdp_prog = priv->xdp_prog ? bpf_prog_inc(priv->xdp_prog) : NULL;
538         if (IS_ERR(rq->xdp_prog)) {
539                 err = PTR_ERR(rq->xdp_prog);
540                 rq->xdp_prog = NULL;
541                 goto err_rq_wq_destroy;
542         }
543
544         if (rq->xdp_prog) {
545                 rq->buff.map_dir = DMA_BIDIRECTIONAL;
546                 rq->rx_headroom = XDP_PACKET_HEADROOM;
547         } else {
548                 rq->buff.map_dir = DMA_FROM_DEVICE;
549                 rq->rx_headroom = MLX5_RX_HEADROOM;
550         }
551
552         switch (priv->params.rq_wq_type) {
553         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
554                 if (mlx5e_is_vf_vport_rep(priv)) {
555                         err = -EINVAL;
556                         goto err_rq_wq_destroy;
557                 }
558
559                 rq->handle_rx_cqe = mlx5e_handle_rx_cqe_mpwrq;
560                 rq->alloc_wqe = mlx5e_alloc_rx_mpwqe;
561                 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
562
563                 rq->mpwqe_stride_sz = BIT(priv->params.mpwqe_log_stride_sz);
564                 rq->mpwqe_num_strides = BIT(priv->params.mpwqe_log_num_strides);
565
566                 rq->buff.wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides;
567                 byte_count = rq->buff.wqe_sz;
568
569                 err = mlx5e_create_rq_umr_mkey(rq);
570                 if (err)
571                         goto err_rq_wq_destroy;
572                 rq->mkey_be = cpu_to_be32(rq->umr_mkey.key);
573
574                 err = mlx5e_rq_alloc_mpwqe_info(rq, c);
575                 if (err)
576                         goto err_destroy_umr_mkey;
577                 break;
578         default: /* MLX5_WQ_TYPE_LINKED_LIST */
579                 rq->dma_info = kzalloc_node(wq_sz * sizeof(*rq->dma_info),
580                                             GFP_KERNEL, cpu_to_node(c->cpu));
581                 if (!rq->dma_info) {
582                         err = -ENOMEM;
583                         goto err_rq_wq_destroy;
584                 }
585
586                 if (mlx5e_is_vf_vport_rep(priv))
587                         rq->handle_rx_cqe = mlx5e_handle_rx_cqe_rep;
588                 else
589                         rq->handle_rx_cqe = mlx5e_handle_rx_cqe;
590
591                 rq->alloc_wqe = mlx5e_alloc_rx_wqe;
592                 rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
593
594                 rq->buff.wqe_sz = (priv->params.lro_en) ?
595                                 priv->params.lro_wqe_sz :
596                                 MLX5E_SW2HW_MTU(priv->netdev->mtu);
597                 byte_count = rq->buff.wqe_sz;
598
599                 /* calc the required page order */
600                 frag_sz = rq->rx_headroom +
601                           byte_count /* packet data */ +
602                           SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
603                 frag_sz = SKB_DATA_ALIGN(frag_sz);
604
605                 npages = DIV_ROUND_UP(frag_sz, PAGE_SIZE);
606                 rq->buff.page_order = order_base_2(npages);
607
608                 byte_count |= MLX5_HW_START_PADDING;
609                 rq->mkey_be = c->mkey_be;
610         }
611
612         for (i = 0; i < wq_sz; i++) {
613                 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
614
615                 wqe->data.byte_count = cpu_to_be32(byte_count);
616                 wqe->data.lkey = rq->mkey_be;
617         }
618
619         INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
620         rq->am.mode = priv->params.rx_cq_period_mode;
621
622         rq->page_cache.head = 0;
623         rq->page_cache.tail = 0;
624
625         return 0;
626
627 err_destroy_umr_mkey:
628         mlx5_core_destroy_mkey(mdev, &rq->umr_mkey);
629
630 err_rq_wq_destroy:
631         if (rq->xdp_prog)
632                 bpf_prog_put(rq->xdp_prog);
633         mlx5_wq_destroy(&rq->wq_ctrl);
634
635         return err;
636 }
637
638 static void mlx5e_destroy_rq(struct mlx5e_rq *rq)
639 {
640         int i;
641
642         if (rq->xdp_prog)
643                 bpf_prog_put(rq->xdp_prog);
644
645         switch (rq->wq_type) {
646         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
647                 mlx5e_rq_free_mpwqe_info(rq);
648                 mlx5_core_destroy_mkey(rq->priv->mdev, &rq->umr_mkey);
649                 break;
650         default: /* MLX5_WQ_TYPE_LINKED_LIST */
651                 kfree(rq->dma_info);
652         }
653
654         for (i = rq->page_cache.head; i != rq->page_cache.tail;
655              i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
656                 struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
657
658                 mlx5e_page_release(rq, dma_info, false);
659         }
660         mlx5_wq_destroy(&rq->wq_ctrl);
661 }
662
663 static int mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
664 {
665         struct mlx5e_priv *priv = rq->priv;
666         struct mlx5_core_dev *mdev = priv->mdev;
667
668         void *in;
669         void *rqc;
670         void *wq;
671         int inlen;
672         int err;
673
674         inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
675                 sizeof(u64) * rq->wq_ctrl.buf.npages;
676         in = mlx5_vzalloc(inlen);
677         if (!in)
678                 return -ENOMEM;
679
680         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
681         wq  = MLX5_ADDR_OF(rqc, rqc, wq);
682
683         memcpy(rqc, param->rqc, sizeof(param->rqc));
684
685         MLX5_SET(rqc,  rqc, cqn,                rq->cq.mcq.cqn);
686         MLX5_SET(rqc,  rqc, state,              MLX5_RQC_STATE_RST);
687         MLX5_SET(rqc,  rqc, vsd, priv->params.vlan_strip_disable);
688         MLX5_SET(wq,   wq,  log_wq_pg_sz,       rq->wq_ctrl.buf.page_shift -
689                                                 MLX5_ADAPTER_PAGE_SHIFT);
690         MLX5_SET64(wq, wq,  dbr_addr,           rq->wq_ctrl.db.dma);
691
692         mlx5_fill_page_array(&rq->wq_ctrl.buf,
693                              (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
694
695         err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
696
697         kvfree(in);
698
699         return err;
700 }
701
702 static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state,
703                                  int next_state)
704 {
705         struct mlx5e_channel *c = rq->channel;
706         struct mlx5e_priv *priv = c->priv;
707         struct mlx5_core_dev *mdev = priv->mdev;
708
709         void *in;
710         void *rqc;
711         int inlen;
712         int err;
713
714         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
715         in = mlx5_vzalloc(inlen);
716         if (!in)
717                 return -ENOMEM;
718
719         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
720
721         MLX5_SET(modify_rq_in, in, rq_state, curr_state);
722         MLX5_SET(rqc, rqc, state, next_state);
723
724         err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
725
726         kvfree(in);
727
728         return err;
729 }
730
731 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
732 {
733         struct mlx5e_channel *c = rq->channel;
734         struct mlx5e_priv *priv = c->priv;
735         struct mlx5_core_dev *mdev = priv->mdev;
736
737         void *in;
738         void *rqc;
739         int inlen;
740         int err;
741
742         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
743         in = mlx5_vzalloc(inlen);
744         if (!in)
745                 return -ENOMEM;
746
747         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
748
749         MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
750         MLX5_SET64(modify_rq_in, in, modify_bitmask,
751                    MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
752         MLX5_SET(rqc, rqc, vsd, vsd);
753         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
754
755         err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
756
757         kvfree(in);
758
759         return err;
760 }
761
762 static void mlx5e_disable_rq(struct mlx5e_rq *rq)
763 {
764         mlx5_core_destroy_rq(rq->priv->mdev, rq->rqn);
765 }
766
767 static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
768 {
769         unsigned long exp_time = jiffies + msecs_to_jiffies(20000);
770         struct mlx5e_channel *c = rq->channel;
771         struct mlx5e_priv *priv = c->priv;
772         struct mlx5_wq_ll *wq = &rq->wq;
773
774         while (time_before(jiffies, exp_time)) {
775                 if (wq->cur_sz >= priv->params.min_rx_wqes)
776                         return 0;
777
778                 msleep(20);
779         }
780
781         return -ETIMEDOUT;
782 }
783
784 static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
785 {
786         struct mlx5_wq_ll *wq = &rq->wq;
787         struct mlx5e_rx_wqe *wqe;
788         __be16 wqe_ix_be;
789         u16 wqe_ix;
790
791         /* UMR WQE (if in progress) is always at wq->head */
792         if (test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
793                 mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
794
795         while (!mlx5_wq_ll_is_empty(wq)) {
796                 wqe_ix_be = *wq->tail_next;
797                 wqe_ix    = be16_to_cpu(wqe_ix_be);
798                 wqe       = mlx5_wq_ll_get_wqe(&rq->wq, wqe_ix);
799                 rq->dealloc_wqe(rq, wqe_ix);
800                 mlx5_wq_ll_pop(&rq->wq, wqe_ix_be,
801                                &wqe->next.next_wqe_index);
802         }
803 }
804
805 static int mlx5e_open_rq(struct mlx5e_channel *c,
806                          struct mlx5e_rq_param *param,
807                          struct mlx5e_rq *rq)
808 {
809         struct mlx5e_sq *sq = &c->icosq;
810         u16 pi = sq->pc & sq->wq.sz_m1;
811         int err;
812
813         err = mlx5e_create_rq(c, param, rq);
814         if (err)
815                 return err;
816
817         err = mlx5e_enable_rq(rq, param);
818         if (err)
819                 goto err_destroy_rq;
820
821         set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
822         err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
823         if (err)
824                 goto err_disable_rq;
825
826         if (param->am_enabled)
827                 set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
828
829         sq->db.ico_wqe[pi].opcode     = MLX5_OPCODE_NOP;
830         sq->db.ico_wqe[pi].num_wqebbs = 1;
831         mlx5e_send_nop(sq, true); /* trigger mlx5e_post_rx_wqes() */
832
833         return 0;
834
835 err_disable_rq:
836         clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
837         mlx5e_disable_rq(rq);
838 err_destroy_rq:
839         mlx5e_destroy_rq(rq);
840
841         return err;
842 }
843
844 static void mlx5e_close_rq(struct mlx5e_rq *rq)
845 {
846         clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
847         napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */
848         cancel_work_sync(&rq->am.work);
849
850         mlx5e_disable_rq(rq);
851         mlx5e_free_rx_descs(rq);
852         mlx5e_destroy_rq(rq);
853 }
854
855 static void mlx5e_free_sq_xdp_db(struct mlx5e_sq *sq)
856 {
857         kfree(sq->db.xdp.di);
858         kfree(sq->db.xdp.wqe_info);
859 }
860
861 static int mlx5e_alloc_sq_xdp_db(struct mlx5e_sq *sq, int numa)
862 {
863         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
864
865         sq->db.xdp.di = kzalloc_node(sizeof(*sq->db.xdp.di) * wq_sz,
866                                      GFP_KERNEL, numa);
867         sq->db.xdp.wqe_info = kzalloc_node(sizeof(*sq->db.xdp.wqe_info) * wq_sz,
868                                            GFP_KERNEL, numa);
869         if (!sq->db.xdp.di || !sq->db.xdp.wqe_info) {
870                 mlx5e_free_sq_xdp_db(sq);
871                 return -ENOMEM;
872         }
873
874         return 0;
875 }
876
877 static void mlx5e_free_sq_ico_db(struct mlx5e_sq *sq)
878 {
879         kfree(sq->db.ico_wqe);
880 }
881
882 static int mlx5e_alloc_sq_ico_db(struct mlx5e_sq *sq, int numa)
883 {
884         u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
885
886         sq->db.ico_wqe = kzalloc_node(sizeof(*sq->db.ico_wqe) * wq_sz,
887                                       GFP_KERNEL, numa);
888         if (!sq->db.ico_wqe)
889                 return -ENOMEM;
890
891         return 0;
892 }
893
894 static void mlx5e_free_sq_txq_db(struct mlx5e_sq *sq)
895 {
896         kfree(sq->db.txq.wqe_info);
897         kfree(sq->db.txq.dma_fifo);
898         kfree(sq->db.txq.skb);
899 }
900
901 static int mlx5e_alloc_sq_txq_db(struct mlx5e_sq *sq, int numa)
902 {
903         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
904         int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
905
906         sq->db.txq.skb = kzalloc_node(wq_sz * sizeof(*sq->db.txq.skb),
907                                       GFP_KERNEL, numa);
908         sq->db.txq.dma_fifo = kzalloc_node(df_sz * sizeof(*sq->db.txq.dma_fifo),
909                                            GFP_KERNEL, numa);
910         sq->db.txq.wqe_info = kzalloc_node(wq_sz * sizeof(*sq->db.txq.wqe_info),
911                                            GFP_KERNEL, numa);
912         if (!sq->db.txq.skb || !sq->db.txq.dma_fifo || !sq->db.txq.wqe_info) {
913                 mlx5e_free_sq_txq_db(sq);
914                 return -ENOMEM;
915         }
916
917         sq->dma_fifo_mask = df_sz - 1;
918
919         return 0;
920 }
921
922 static void mlx5e_free_sq_db(struct mlx5e_sq *sq)
923 {
924         switch (sq->type) {
925         case MLX5E_SQ_TXQ:
926                 mlx5e_free_sq_txq_db(sq);
927                 break;
928         case MLX5E_SQ_ICO:
929                 mlx5e_free_sq_ico_db(sq);
930                 break;
931         case MLX5E_SQ_XDP:
932                 mlx5e_free_sq_xdp_db(sq);
933                 break;
934         }
935 }
936
937 static int mlx5e_alloc_sq_db(struct mlx5e_sq *sq, int numa)
938 {
939         switch (sq->type) {
940         case MLX5E_SQ_TXQ:
941                 return mlx5e_alloc_sq_txq_db(sq, numa);
942         case MLX5E_SQ_ICO:
943                 return mlx5e_alloc_sq_ico_db(sq, numa);
944         case MLX5E_SQ_XDP:
945                 return mlx5e_alloc_sq_xdp_db(sq, numa);
946         }
947
948         return 0;
949 }
950
951 static int mlx5e_sq_get_max_wqebbs(u8 sq_type)
952 {
953         switch (sq_type) {
954         case MLX5E_SQ_ICO:
955                 return MLX5E_ICOSQ_MAX_WQEBBS;
956         case MLX5E_SQ_XDP:
957                 return MLX5E_XDP_TX_WQEBBS;
958         }
959         return MLX5_SEND_WQE_MAX_WQEBBS;
960 }
961
962 static int mlx5e_create_sq(struct mlx5e_channel *c,
963                            int tc,
964                            struct mlx5e_sq_param *param,
965                            struct mlx5e_sq *sq)
966 {
967         struct mlx5e_priv *priv = c->priv;
968         struct mlx5_core_dev *mdev = priv->mdev;
969
970         void *sqc = param->sqc;
971         void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
972         int err;
973
974         sq->type      = param->type;
975         sq->pdev      = c->pdev;
976         sq->tstamp    = &priv->tstamp;
977         sq->mkey_be   = c->mkey_be;
978         sq->channel   = c;
979         sq->tc        = tc;
980
981         err = mlx5_alloc_bfreg(mdev, &sq->bfreg, MLX5_CAP_GEN(mdev, bf), false);
982         if (err)
983                 return err;
984
985         param->wq.db_numa_node = cpu_to_node(c->cpu);
986
987         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
988                                  &sq->wq_ctrl);
989         if (err)
990                 goto err_unmap_free_uar;
991
992         sq->wq.db       = &sq->wq.db[MLX5_SND_DBR];
993         if (sq->bfreg.wc)
994                 set_bit(MLX5E_SQ_STATE_BF_ENABLE, &sq->state);
995
996         sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
997         sq->max_inline  = param->max_inline;
998         sq->min_inline_mode =
999                 MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT ?
1000                 param->min_inline_mode : 0;
1001
1002         err = mlx5e_alloc_sq_db(sq, cpu_to_node(c->cpu));
1003         if (err)
1004                 goto err_sq_wq_destroy;
1005
1006         if (sq->type == MLX5E_SQ_TXQ) {
1007                 int txq_ix;
1008
1009                 txq_ix = c->ix + tc * priv->params.num_channels;
1010                 sq->txq = netdev_get_tx_queue(priv->netdev, txq_ix);
1011                 priv->txq_to_sq_map[txq_ix] = sq;
1012         }
1013
1014         sq->edge = (sq->wq.sz_m1 + 1) - mlx5e_sq_get_max_wqebbs(sq->type);
1015         sq->bf_budget = MLX5E_SQ_BF_BUDGET;
1016
1017         return 0;
1018
1019 err_sq_wq_destroy:
1020         mlx5_wq_destroy(&sq->wq_ctrl);
1021
1022 err_unmap_free_uar:
1023         mlx5_free_bfreg(mdev, &sq->bfreg);
1024
1025         return err;
1026 }
1027
1028 static void mlx5e_destroy_sq(struct mlx5e_sq *sq)
1029 {
1030         struct mlx5e_channel *c = sq->channel;
1031         struct mlx5e_priv *priv = c->priv;
1032
1033         mlx5e_free_sq_db(sq);
1034         mlx5_wq_destroy(&sq->wq_ctrl);
1035         mlx5_free_bfreg(priv->mdev, &sq->bfreg);
1036 }
1037
1038 static int mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param)
1039 {
1040         struct mlx5e_channel *c = sq->channel;
1041         struct mlx5e_priv *priv = c->priv;
1042         struct mlx5_core_dev *mdev = priv->mdev;
1043
1044         void *in;
1045         void *sqc;
1046         void *wq;
1047         int inlen;
1048         int err;
1049
1050         inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1051                 sizeof(u64) * sq->wq_ctrl.buf.npages;
1052         in = mlx5_vzalloc(inlen);
1053         if (!in)
1054                 return -ENOMEM;
1055
1056         sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1057         wq = MLX5_ADDR_OF(sqc, sqc, wq);
1058
1059         memcpy(sqc, param->sqc, sizeof(param->sqc));
1060
1061         MLX5_SET(sqc,  sqc, tis_num_0, param->type == MLX5E_SQ_ICO ?
1062                                        0 : priv->tisn[sq->tc]);
1063         MLX5_SET(sqc,  sqc, cqn,                sq->cq.mcq.cqn);
1064         MLX5_SET(sqc,  sqc, min_wqe_inline_mode, sq->min_inline_mode);
1065         MLX5_SET(sqc,  sqc, state,              MLX5_SQC_STATE_RST);
1066         MLX5_SET(sqc,  sqc, tis_lst_sz, param->type == MLX5E_SQ_ICO ? 0 : 1);
1067
1068         MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
1069         MLX5_SET(wq,   wq, uar_page,      sq->bfreg.index);
1070         MLX5_SET(wq,   wq, log_wq_pg_sz,  sq->wq_ctrl.buf.page_shift -
1071                                           MLX5_ADAPTER_PAGE_SHIFT);
1072         MLX5_SET64(wq, wq, dbr_addr,      sq->wq_ctrl.db.dma);
1073
1074         mlx5_fill_page_array(&sq->wq_ctrl.buf,
1075                              (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
1076
1077         err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);
1078
1079         kvfree(in);
1080
1081         return err;
1082 }
1083
1084 static int mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state,
1085                            int next_state, bool update_rl, int rl_index)
1086 {
1087         struct mlx5e_channel *c = sq->channel;
1088         struct mlx5e_priv *priv = c->priv;
1089         struct mlx5_core_dev *mdev = priv->mdev;
1090
1091         void *in;
1092         void *sqc;
1093         int inlen;
1094         int err;
1095
1096         inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1097         in = mlx5_vzalloc(inlen);
1098         if (!in)
1099                 return -ENOMEM;
1100
1101         sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1102
1103         MLX5_SET(modify_sq_in, in, sq_state, curr_state);
1104         MLX5_SET(sqc, sqc, state, next_state);
1105         if (update_rl && next_state == MLX5_SQC_STATE_RDY) {
1106                 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
1107                 MLX5_SET(sqc,  sqc, packet_pacing_rate_limit_index, rl_index);
1108         }
1109
1110         err = mlx5_core_modify_sq(mdev, sq->sqn, in, inlen);
1111
1112         kvfree(in);
1113
1114         return err;
1115 }
1116
1117 static void mlx5e_disable_sq(struct mlx5e_sq *sq)
1118 {
1119         struct mlx5e_channel *c = sq->channel;
1120         struct mlx5e_priv *priv = c->priv;
1121         struct mlx5_core_dev *mdev = priv->mdev;
1122
1123         mlx5_core_destroy_sq(mdev, sq->sqn);
1124         if (sq->rate_limit)
1125                 mlx5_rl_remove_rate(mdev, sq->rate_limit);
1126 }
1127
1128 static int mlx5e_open_sq(struct mlx5e_channel *c,
1129                          int tc,
1130                          struct mlx5e_sq_param *param,
1131                          struct mlx5e_sq *sq)
1132 {
1133         int err;
1134
1135         err = mlx5e_create_sq(c, tc, param, sq);
1136         if (err)
1137                 return err;
1138
1139         err = mlx5e_enable_sq(sq, param);
1140         if (err)
1141                 goto err_destroy_sq;
1142
1143         set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1144         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
1145                               false, 0);
1146         if (err)
1147                 goto err_disable_sq;
1148
1149         if (sq->txq) {
1150                 netdev_tx_reset_queue(sq->txq);
1151                 netif_tx_start_queue(sq->txq);
1152         }
1153
1154         return 0;
1155
1156 err_disable_sq:
1157         clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1158         mlx5e_disable_sq(sq);
1159 err_destroy_sq:
1160         mlx5e_destroy_sq(sq);
1161
1162         return err;
1163 }
1164
1165 static inline void netif_tx_disable_queue(struct netdev_queue *txq)
1166 {
1167         __netif_tx_lock_bh(txq);
1168         netif_tx_stop_queue(txq);
1169         __netif_tx_unlock_bh(txq);
1170 }
1171
1172 static void mlx5e_close_sq(struct mlx5e_sq *sq)
1173 {
1174         clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1175         /* prevent netif_tx_wake_queue */
1176         napi_synchronize(&sq->channel->napi);
1177
1178         if (sq->txq) {
1179                 netif_tx_disable_queue(sq->txq);
1180
1181                 /* last doorbell out, godspeed .. */
1182                 if (mlx5e_sq_has_room_for(sq, 1)) {
1183                         sq->db.txq.skb[(sq->pc & sq->wq.sz_m1)] = NULL;
1184                         mlx5e_send_nop(sq, true);
1185                 }
1186         }
1187
1188         mlx5e_disable_sq(sq);
1189         mlx5e_free_sq_descs(sq);
1190         mlx5e_destroy_sq(sq);
1191 }
1192
1193 static int mlx5e_create_cq(struct mlx5e_channel *c,
1194                            struct mlx5e_cq_param *param,
1195                            struct mlx5e_cq *cq)
1196 {
1197         struct mlx5e_priv *priv = c->priv;
1198         struct mlx5_core_dev *mdev = priv->mdev;
1199         struct mlx5_core_cq *mcq = &cq->mcq;
1200         int eqn_not_used;
1201         unsigned int irqn;
1202         int err;
1203         u32 i;
1204
1205         param->wq.buf_numa_node = cpu_to_node(c->cpu);
1206         param->wq.db_numa_node  = cpu_to_node(c->cpu);
1207         param->eq_ix   = c->ix;
1208
1209         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1210                                &cq->wq_ctrl);
1211         if (err)
1212                 return err;
1213
1214         mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
1215
1216         cq->napi        = &c->napi;
1217
1218         mcq->cqe_sz     = 64;
1219         mcq->set_ci_db  = cq->wq_ctrl.db.db;
1220         mcq->arm_db     = cq->wq_ctrl.db.db + 1;
1221         *mcq->set_ci_db = 0;
1222         *mcq->arm_db    = 0;
1223         mcq->vector     = param->eq_ix;
1224         mcq->comp       = mlx5e_completion_event;
1225         mcq->event      = mlx5e_cq_error_event;
1226         mcq->irqn       = irqn;
1227
1228         for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1229                 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1230
1231                 cqe->op_own = 0xf1;
1232         }
1233
1234         cq->channel = c;
1235         cq->priv = priv;
1236
1237         return 0;
1238 }
1239
1240 static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
1241 {
1242         mlx5_cqwq_destroy(&cq->wq_ctrl);
1243 }
1244
1245 static int mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
1246 {
1247         struct mlx5e_priv *priv = cq->priv;
1248         struct mlx5_core_dev *mdev = priv->mdev;
1249         struct mlx5_core_cq *mcq = &cq->mcq;
1250
1251         void *in;
1252         void *cqc;
1253         int inlen;
1254         unsigned int irqn_not_used;
1255         int eqn;
1256         int err;
1257
1258         inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1259                 sizeof(u64) * cq->wq_ctrl.frag_buf.npages;
1260         in = mlx5_vzalloc(inlen);
1261         if (!in)
1262                 return -ENOMEM;
1263
1264         cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1265
1266         memcpy(cqc, param->cqc, sizeof(param->cqc));
1267
1268         mlx5_fill_page_frag_array(&cq->wq_ctrl.frag_buf,
1269                                   (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
1270
1271         mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
1272
1273         MLX5_SET(cqc,   cqc, cq_period_mode, param->cq_period_mode);
1274         MLX5_SET(cqc,   cqc, c_eqn,         eqn);
1275         MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.uar->index);
1276         MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.frag_buf.page_shift -
1277                                             MLX5_ADAPTER_PAGE_SHIFT);
1278         MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
1279
1280         err = mlx5_core_create_cq(mdev, mcq, in, inlen);
1281
1282         kvfree(in);
1283
1284         if (err)
1285                 return err;
1286
1287         mlx5e_cq_arm(cq);
1288
1289         return 0;
1290 }
1291
1292 static void mlx5e_disable_cq(struct mlx5e_cq *cq)
1293 {
1294         struct mlx5e_priv *priv = cq->priv;
1295         struct mlx5_core_dev *mdev = priv->mdev;
1296
1297         mlx5_core_destroy_cq(mdev, &cq->mcq);
1298 }
1299
1300 static int mlx5e_open_cq(struct mlx5e_channel *c,
1301                          struct mlx5e_cq_param *param,
1302                          struct mlx5e_cq *cq,
1303                          struct mlx5e_cq_moder moderation)
1304 {
1305         int err;
1306         struct mlx5e_priv *priv = c->priv;
1307         struct mlx5_core_dev *mdev = priv->mdev;
1308
1309         err = mlx5e_create_cq(c, param, cq);
1310         if (err)
1311                 return err;
1312
1313         err = mlx5e_enable_cq(cq, param);
1314         if (err)
1315                 goto err_destroy_cq;
1316
1317         if (MLX5_CAP_GEN(mdev, cq_moderation))
1318                 mlx5_core_modify_cq_moderation(mdev, &cq->mcq,
1319                                                moderation.usec,
1320                                                moderation.pkts);
1321         return 0;
1322
1323 err_destroy_cq:
1324         mlx5e_destroy_cq(cq);
1325
1326         return err;
1327 }
1328
1329 static void mlx5e_close_cq(struct mlx5e_cq *cq)
1330 {
1331         mlx5e_disable_cq(cq);
1332         mlx5e_destroy_cq(cq);
1333 }
1334
1335 static int mlx5e_get_cpu(struct mlx5e_priv *priv, int ix)
1336 {
1337         return cpumask_first(priv->mdev->priv.irq_info[ix].mask);
1338 }
1339
1340 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
1341                              struct mlx5e_channel_param *cparam)
1342 {
1343         struct mlx5e_priv *priv = c->priv;
1344         int err;
1345         int tc;
1346
1347         for (tc = 0; tc < c->num_tc; tc++) {
1348                 err = mlx5e_open_cq(c, &cparam->tx_cq, &c->sq[tc].cq,
1349                                     priv->params.tx_cq_moderation);
1350                 if (err)
1351                         goto err_close_tx_cqs;
1352         }
1353
1354         return 0;
1355
1356 err_close_tx_cqs:
1357         for (tc--; tc >= 0; tc--)
1358                 mlx5e_close_cq(&c->sq[tc].cq);
1359
1360         return err;
1361 }
1362
1363 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
1364 {
1365         int tc;
1366
1367         for (tc = 0; tc < c->num_tc; tc++)
1368                 mlx5e_close_cq(&c->sq[tc].cq);
1369 }
1370
1371 static int mlx5e_open_sqs(struct mlx5e_channel *c,
1372                           struct mlx5e_channel_param *cparam)
1373 {
1374         int err;
1375         int tc;
1376
1377         for (tc = 0; tc < c->num_tc; tc++) {
1378                 err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
1379                 if (err)
1380                         goto err_close_sqs;
1381         }
1382
1383         return 0;
1384
1385 err_close_sqs:
1386         for (tc--; tc >= 0; tc--)
1387                 mlx5e_close_sq(&c->sq[tc]);
1388
1389         return err;
1390 }
1391
1392 static void mlx5e_close_sqs(struct mlx5e_channel *c)
1393 {
1394         int tc;
1395
1396         for (tc = 0; tc < c->num_tc; tc++)
1397                 mlx5e_close_sq(&c->sq[tc]);
1398 }
1399
1400 static void mlx5e_build_channeltc_to_txq_map(struct mlx5e_priv *priv, int ix)
1401 {
1402         int i;
1403
1404         for (i = 0; i < priv->profile->max_tc; i++)
1405                 priv->channeltc_to_txq_map[ix][i] =
1406                         ix + i * priv->params.num_channels;
1407 }
1408
1409 static int mlx5e_set_sq_maxrate(struct net_device *dev,
1410                                 struct mlx5e_sq *sq, u32 rate)
1411 {
1412         struct mlx5e_priv *priv = netdev_priv(dev);
1413         struct mlx5_core_dev *mdev = priv->mdev;
1414         u16 rl_index = 0;
1415         int err;
1416
1417         if (rate == sq->rate_limit)
1418                 /* nothing to do */
1419                 return 0;
1420
1421         if (sq->rate_limit)
1422                 /* remove current rl index to free space to next ones */
1423                 mlx5_rl_remove_rate(mdev, sq->rate_limit);
1424
1425         sq->rate_limit = 0;
1426
1427         if (rate) {
1428                 err = mlx5_rl_add_rate(mdev, rate, &rl_index);
1429                 if (err) {
1430                         netdev_err(dev, "Failed configuring rate %u: %d\n",
1431                                    rate, err);
1432                         return err;
1433                 }
1434         }
1435
1436         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY,
1437                               MLX5_SQC_STATE_RDY, true, rl_index);
1438         if (err) {
1439                 netdev_err(dev, "Failed configuring rate %u: %d\n",
1440                            rate, err);
1441                 /* remove the rate from the table */
1442                 if (rate)
1443                         mlx5_rl_remove_rate(mdev, rate);
1444                 return err;
1445         }
1446
1447         sq->rate_limit = rate;
1448         return 0;
1449 }
1450
1451 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
1452 {
1453         struct mlx5e_priv *priv = netdev_priv(dev);
1454         struct mlx5_core_dev *mdev = priv->mdev;
1455         struct mlx5e_sq *sq = priv->txq_to_sq_map[index];
1456         int err = 0;
1457
1458         if (!mlx5_rl_is_supported(mdev)) {
1459                 netdev_err(dev, "Rate limiting is not supported on this device\n");
1460                 return -EINVAL;
1461         }
1462
1463         /* rate is given in Mb/sec, HW config is in Kb/sec */
1464         rate = rate << 10;
1465
1466         /* Check whether rate in valid range, 0 is always valid */
1467         if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
1468                 netdev_err(dev, "TX rate %u, is not in range\n", rate);
1469                 return -ERANGE;
1470         }
1471
1472         mutex_lock(&priv->state_lock);
1473         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
1474                 err = mlx5e_set_sq_maxrate(dev, sq, rate);
1475         if (!err)
1476                 priv->tx_rates[index] = rate;
1477         mutex_unlock(&priv->state_lock);
1478
1479         return err;
1480 }
1481
1482 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
1483                               struct mlx5e_channel_param *cparam,
1484                               struct mlx5e_channel **cp)
1485 {
1486         struct mlx5e_cq_moder icosq_cq_moder = {0, 0};
1487         struct net_device *netdev = priv->netdev;
1488         struct mlx5e_cq_moder rx_cq_profile;
1489         int cpu = mlx5e_get_cpu(priv, ix);
1490         struct mlx5e_channel *c;
1491         struct mlx5e_sq *sq;
1492         int err;
1493         int i;
1494
1495         c = kzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
1496         if (!c)
1497                 return -ENOMEM;
1498
1499         c->priv     = priv;
1500         c->ix       = ix;
1501         c->cpu      = cpu;
1502         c->pdev     = &priv->mdev->pdev->dev;
1503         c->netdev   = priv->netdev;
1504         c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
1505         c->num_tc   = priv->params.num_tc;
1506         c->xdp      = !!priv->xdp_prog;
1507
1508         if (priv->params.rx_am_enabled)
1509                 rx_cq_profile = mlx5e_am_get_def_profile(priv->params.rx_cq_period_mode);
1510         else
1511                 rx_cq_profile = priv->params.rx_cq_moderation;
1512
1513         mlx5e_build_channeltc_to_txq_map(priv, ix);
1514
1515         netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
1516
1517         err = mlx5e_open_cq(c, &cparam->icosq_cq, &c->icosq.cq, icosq_cq_moder);
1518         if (err)
1519                 goto err_napi_del;
1520
1521         err = mlx5e_open_tx_cqs(c, cparam);
1522         if (err)
1523                 goto err_close_icosq_cq;
1524
1525         err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq,
1526                             rx_cq_profile);
1527         if (err)
1528                 goto err_close_tx_cqs;
1529
1530         /* XDP SQ CQ params are same as normal TXQ sq CQ params */
1531         err = c->xdp ? mlx5e_open_cq(c, &cparam->tx_cq, &c->xdp_sq.cq,
1532                                      priv->params.tx_cq_moderation) : 0;
1533         if (err)
1534                 goto err_close_rx_cq;
1535
1536         napi_enable(&c->napi);
1537
1538         err = mlx5e_open_sq(c, 0, &cparam->icosq, &c->icosq);
1539         if (err)
1540                 goto err_disable_napi;
1541
1542         err = mlx5e_open_sqs(c, cparam);
1543         if (err)
1544                 goto err_close_icosq;
1545
1546         for (i = 0; i < priv->params.num_tc; i++) {
1547                 u32 txq_ix = priv->channeltc_to_txq_map[ix][i];
1548
1549                 if (priv->tx_rates[txq_ix]) {
1550                         sq = priv->txq_to_sq_map[txq_ix];
1551                         mlx5e_set_sq_maxrate(priv->netdev, sq,
1552                                              priv->tx_rates[txq_ix]);
1553                 }
1554         }
1555
1556         err = c->xdp ? mlx5e_open_sq(c, 0, &cparam->xdp_sq, &c->xdp_sq) : 0;
1557         if (err)
1558                 goto err_close_sqs;
1559
1560         err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
1561         if (err)
1562                 goto err_close_xdp_sq;
1563
1564         netif_set_xps_queue(netdev, get_cpu_mask(c->cpu), ix);
1565         *cp = c;
1566
1567         return 0;
1568 err_close_xdp_sq:
1569         if (c->xdp)
1570                 mlx5e_close_sq(&c->xdp_sq);
1571
1572 err_close_sqs:
1573         mlx5e_close_sqs(c);
1574
1575 err_close_icosq:
1576         mlx5e_close_sq(&c->icosq);
1577
1578 err_disable_napi:
1579         napi_disable(&c->napi);
1580         if (c->xdp)
1581                 mlx5e_close_cq(&c->xdp_sq.cq);
1582
1583 err_close_rx_cq:
1584         mlx5e_close_cq(&c->rq.cq);
1585
1586 err_close_tx_cqs:
1587         mlx5e_close_tx_cqs(c);
1588
1589 err_close_icosq_cq:
1590         mlx5e_close_cq(&c->icosq.cq);
1591
1592 err_napi_del:
1593         netif_napi_del(&c->napi);
1594         kfree(c);
1595
1596         return err;
1597 }
1598
1599 static void mlx5e_close_channel(struct mlx5e_channel *c)
1600 {
1601         mlx5e_close_rq(&c->rq);
1602         if (c->xdp)
1603                 mlx5e_close_sq(&c->xdp_sq);
1604         mlx5e_close_sqs(c);
1605         mlx5e_close_sq(&c->icosq);
1606         napi_disable(&c->napi);
1607         if (c->xdp)
1608                 mlx5e_close_cq(&c->xdp_sq.cq);
1609         mlx5e_close_cq(&c->rq.cq);
1610         mlx5e_close_tx_cqs(c);
1611         mlx5e_close_cq(&c->icosq.cq);
1612         netif_napi_del(&c->napi);
1613
1614         kfree(c);
1615 }
1616
1617 static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
1618                                  struct mlx5e_rq_param *param)
1619 {
1620         void *rqc = param->rqc;
1621         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1622
1623         switch (priv->params.rq_wq_type) {
1624         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1625                 MLX5_SET(wq, wq, log_wqe_num_of_strides,
1626                          priv->params.mpwqe_log_num_strides - 9);
1627                 MLX5_SET(wq, wq, log_wqe_stride_size,
1628                          priv->params.mpwqe_log_stride_sz - 6);
1629                 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
1630                 break;
1631         default: /* MLX5_WQ_TYPE_LINKED_LIST */
1632                 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1633         }
1634
1635         MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1636         MLX5_SET(wq, wq, log_wq_stride,    ilog2(sizeof(struct mlx5e_rx_wqe)));
1637         MLX5_SET(wq, wq, log_wq_sz,        priv->params.log_rq_size);
1638         MLX5_SET(wq, wq, pd,               priv->mdev->mlx5e_res.pdn);
1639         MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter);
1640
1641         param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
1642         param->wq.linear = 1;
1643
1644         param->am_enabled = priv->params.rx_am_enabled;
1645 }
1646
1647 static void mlx5e_build_drop_rq_param(struct mlx5e_rq_param *param)
1648 {
1649         void *rqc = param->rqc;
1650         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1651
1652         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1653         MLX5_SET(wq, wq, log_wq_stride,    ilog2(sizeof(struct mlx5e_rx_wqe)));
1654 }
1655
1656 static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
1657                                         struct mlx5e_sq_param *param)
1658 {
1659         void *sqc = param->sqc;
1660         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1661
1662         MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1663         MLX5_SET(wq, wq, pd,            priv->mdev->mlx5e_res.pdn);
1664
1665         param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
1666 }
1667
1668 static void mlx5e_build_sq_param(struct mlx5e_priv *priv,
1669                                  struct mlx5e_sq_param *param)
1670 {
1671         void *sqc = param->sqc;
1672         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1673
1674         mlx5e_build_sq_param_common(priv, param);
1675         MLX5_SET(wq, wq, log_wq_sz,     priv->params.log_sq_size);
1676
1677         param->max_inline = priv->params.tx_max_inline;
1678         param->min_inline_mode = priv->params.tx_min_inline_mode;
1679         param->type = MLX5E_SQ_TXQ;
1680 }
1681
1682 static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
1683                                         struct mlx5e_cq_param *param)
1684 {
1685         void *cqc = param->cqc;
1686
1687         MLX5_SET(cqc, cqc, uar_page, priv->mdev->priv.uar->index);
1688 }
1689
1690 static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
1691                                     struct mlx5e_cq_param *param)
1692 {
1693         void *cqc = param->cqc;
1694         u8 log_cq_size;
1695
1696         switch (priv->params.rq_wq_type) {
1697         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
1698                 log_cq_size = priv->params.log_rq_size +
1699                         priv->params.mpwqe_log_num_strides;
1700                 break;
1701         default: /* MLX5_WQ_TYPE_LINKED_LIST */
1702                 log_cq_size = priv->params.log_rq_size;
1703         }
1704
1705         MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
1706         if (MLX5E_GET_PFLAG(priv, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
1707                 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM);
1708                 MLX5_SET(cqc, cqc, cqe_comp_en, 1);
1709         }
1710
1711         mlx5e_build_common_cq_param(priv, param);
1712
1713         param->cq_period_mode = priv->params.rx_cq_period_mode;
1714 }
1715
1716 static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
1717                                     struct mlx5e_cq_param *param)
1718 {
1719         void *cqc = param->cqc;
1720
1721         MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
1722
1723         mlx5e_build_common_cq_param(priv, param);
1724
1725         param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1726 }
1727
1728 static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
1729                                      struct mlx5e_cq_param *param,
1730                                      u8 log_wq_size)
1731 {
1732         void *cqc = param->cqc;
1733
1734         MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
1735
1736         mlx5e_build_common_cq_param(priv, param);
1737
1738         param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
1739 }
1740
1741 static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
1742                                     struct mlx5e_sq_param *param,
1743                                     u8 log_wq_size)
1744 {
1745         void *sqc = param->sqc;
1746         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1747
1748         mlx5e_build_sq_param_common(priv, param);
1749
1750         MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
1751         MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));
1752
1753         param->type = MLX5E_SQ_ICO;
1754 }
1755
1756 static void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv,
1757                                     struct mlx5e_sq_param *param)
1758 {
1759         void *sqc = param->sqc;
1760         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1761
1762         mlx5e_build_sq_param_common(priv, param);
1763         MLX5_SET(wq, wq, log_wq_sz,     priv->params.log_sq_size);
1764
1765         param->max_inline = priv->params.tx_max_inline;
1766         /* FOR XDP SQs will support only L2 inline mode */
1767         param->min_inline_mode = MLX5_INLINE_MODE_NONE;
1768         param->type = MLX5E_SQ_XDP;
1769 }
1770
1771 static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_channel_param *cparam)
1772 {
1773         u8 icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
1774
1775         mlx5e_build_rq_param(priv, &cparam->rq);
1776         mlx5e_build_sq_param(priv, &cparam->sq);
1777         mlx5e_build_xdpsq_param(priv, &cparam->xdp_sq);
1778         mlx5e_build_icosq_param(priv, &cparam->icosq, icosq_log_wq_sz);
1779         mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
1780         mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
1781         mlx5e_build_ico_cq_param(priv, &cparam->icosq_cq, icosq_log_wq_sz);
1782 }
1783
1784 static int mlx5e_open_channels(struct mlx5e_priv *priv)
1785 {
1786         struct mlx5e_channel_param *cparam;
1787         int nch = priv->params.num_channels;
1788         int err = -ENOMEM;
1789         int i;
1790         int j;
1791
1792         priv->channel = kcalloc(nch, sizeof(struct mlx5e_channel *),
1793                                 GFP_KERNEL);
1794
1795         priv->txq_to_sq_map = kcalloc(nch * priv->params.num_tc,
1796                                       sizeof(struct mlx5e_sq *), GFP_KERNEL);
1797
1798         cparam = kzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
1799
1800         if (!priv->channel || !priv->txq_to_sq_map || !cparam)
1801                 goto err_free_txq_to_sq_map;
1802
1803         mlx5e_build_channel_param(priv, cparam);
1804
1805         for (i = 0; i < nch; i++) {
1806                 err = mlx5e_open_channel(priv, i, cparam, &priv->channel[i]);
1807                 if (err)
1808                         goto err_close_channels;
1809         }
1810
1811         for (j = 0; j < nch; j++) {
1812                 err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j]->rq);
1813                 if (err)
1814                         goto err_close_channels;
1815         }
1816
1817         /* FIXME: This is a W/A for tx timeout watch dog false alarm when
1818          * polling for inactive tx queues.
1819          */
1820         netif_tx_start_all_queues(priv->netdev);
1821
1822         kfree(cparam);
1823         return 0;
1824
1825 err_close_channels:
1826         for (i--; i >= 0; i--)
1827                 mlx5e_close_channel(priv->channel[i]);
1828
1829 err_free_txq_to_sq_map:
1830         kfree(priv->txq_to_sq_map);
1831         kfree(priv->channel);
1832         kfree(cparam);
1833
1834         return err;
1835 }
1836
1837 static void mlx5e_close_channels(struct mlx5e_priv *priv)
1838 {
1839         int i;
1840
1841         /* FIXME: This is a W/A only for tx timeout watch dog false alarm when
1842          * polling for inactive tx queues.
1843          */
1844         netif_tx_stop_all_queues(priv->netdev);
1845         netif_tx_disable(priv->netdev);
1846
1847         for (i = 0; i < priv->params.num_channels; i++)
1848                 mlx5e_close_channel(priv->channel[i]);
1849
1850         kfree(priv->txq_to_sq_map);
1851         kfree(priv->channel);
1852 }
1853
1854 static int mlx5e_rx_hash_fn(int hfunc)
1855 {
1856         return (hfunc == ETH_RSS_HASH_TOP) ?
1857                MLX5_RX_HASH_FN_TOEPLITZ :
1858                MLX5_RX_HASH_FN_INVERTED_XOR8;
1859 }
1860
1861 static int mlx5e_bits_invert(unsigned long a, int size)
1862 {
1863         int inv = 0;
1864         int i;
1865
1866         for (i = 0; i < size; i++)
1867                 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
1868
1869         return inv;
1870 }
1871
1872 static void mlx5e_fill_indir_rqt_rqns(struct mlx5e_priv *priv, void *rqtc)
1873 {
1874         int i;
1875
1876         for (i = 0; i < MLX5E_INDIR_RQT_SIZE; i++) {
1877                 int ix = i;
1878                 u32 rqn;
1879
1880                 if (priv->params.rss_hfunc == ETH_RSS_HASH_XOR)
1881                         ix = mlx5e_bits_invert(i, MLX5E_LOG_INDIR_RQT_SIZE);
1882
1883                 ix = priv->params.indirection_rqt[ix];
1884                 rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
1885                                 priv->channel[ix]->rq.rqn :
1886                                 priv->drop_rq.rqn;
1887                 MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
1888         }
1889 }
1890
1891 static void mlx5e_fill_direct_rqt_rqn(struct mlx5e_priv *priv, void *rqtc,
1892                                       int ix)
1893 {
1894         u32 rqn = test_bit(MLX5E_STATE_OPENED, &priv->state) ?
1895                         priv->channel[ix]->rq.rqn :
1896                         priv->drop_rq.rqn;
1897
1898         MLX5_SET(rqtc, rqtc, rq_num[0], rqn);
1899 }
1900
1901 static int mlx5e_create_rqt(struct mlx5e_priv *priv, int sz,
1902                             int ix, struct mlx5e_rqt *rqt)
1903 {
1904         struct mlx5_core_dev *mdev = priv->mdev;
1905         void *rqtc;
1906         int inlen;
1907         int err;
1908         u32 *in;
1909
1910         inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1911         in = mlx5_vzalloc(inlen);
1912         if (!in)
1913                 return -ENOMEM;
1914
1915         rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
1916
1917         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1918         MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
1919
1920         if (sz > 1) /* RSS */
1921                 mlx5e_fill_indir_rqt_rqns(priv, rqtc);
1922         else
1923                 mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
1924
1925         err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn);
1926         if (!err)
1927                 rqt->enabled = true;
1928
1929         kvfree(in);
1930         return err;
1931 }
1932
1933 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt)
1934 {
1935         rqt->enabled = false;
1936         mlx5_core_destroy_rqt(priv->mdev, rqt->rqtn);
1937 }
1938
1939 static int mlx5e_create_indirect_rqts(struct mlx5e_priv *priv)
1940 {
1941         struct mlx5e_rqt *rqt = &priv->indir_rqt;
1942
1943         return mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, 0, rqt);
1944 }
1945
1946 int mlx5e_create_direct_rqts(struct mlx5e_priv *priv)
1947 {
1948         struct mlx5e_rqt *rqt;
1949         int err;
1950         int ix;
1951
1952         for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
1953                 rqt = &priv->direct_tir[ix].rqt;
1954                 err = mlx5e_create_rqt(priv, 1 /*size */, ix, rqt);
1955                 if (err)
1956                         goto err_destroy_rqts;
1957         }
1958
1959         return 0;
1960
1961 err_destroy_rqts:
1962         for (ix--; ix >= 0; ix--)
1963                 mlx5e_destroy_rqt(priv, &priv->direct_tir[ix].rqt);
1964
1965         return err;
1966 }
1967
1968 int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, int ix)
1969 {
1970         struct mlx5_core_dev *mdev = priv->mdev;
1971         void *rqtc;
1972         int inlen;
1973         u32 *in;
1974         int err;
1975
1976         inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
1977         in = mlx5_vzalloc(inlen);
1978         if (!in)
1979                 return -ENOMEM;
1980
1981         rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
1982
1983         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1984         if (sz > 1) /* RSS */
1985                 mlx5e_fill_indir_rqt_rqns(priv, rqtc);
1986         else
1987                 mlx5e_fill_direct_rqt_rqn(priv, rqtc, ix);
1988
1989         MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
1990
1991         err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen);
1992
1993         kvfree(in);
1994
1995         return err;
1996 }
1997
1998 static void mlx5e_redirect_rqts(struct mlx5e_priv *priv)
1999 {
2000         u32 rqtn;
2001         int ix;
2002
2003         if (priv->indir_rqt.enabled) {
2004                 rqtn = priv->indir_rqt.rqtn;
2005                 mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, 0);
2006         }
2007
2008         for (ix = 0; ix < priv->params.num_channels; ix++) {
2009                 if (!priv->direct_tir[ix].rqt.enabled)
2010                         continue;
2011                 rqtn = priv->direct_tir[ix].rqt.rqtn;
2012                 mlx5e_redirect_rqt(priv, rqtn, 1, ix);
2013         }
2014 }
2015
2016 static void mlx5e_build_tir_ctx_lro(void *tirc, struct mlx5e_priv *priv)
2017 {
2018         if (!priv->params.lro_en)
2019                 return;
2020
2021 #define ROUGH_MAX_L2_L3_HDR_SZ 256
2022
2023         MLX5_SET(tirc, tirc, lro_enable_mask,
2024                  MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
2025                  MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
2026         MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
2027                  (priv->params.lro_wqe_sz -
2028                   ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
2029         MLX5_SET(tirc, tirc, lro_timeout_period_usecs, priv->params.lro_timeout);
2030 }
2031
2032 void mlx5e_build_tir_ctx_hash(void *tirc, struct mlx5e_priv *priv)
2033 {
2034         MLX5_SET(tirc, tirc, rx_hash_fn,
2035                  mlx5e_rx_hash_fn(priv->params.rss_hfunc));
2036         if (priv->params.rss_hfunc == ETH_RSS_HASH_TOP) {
2037                 void *rss_key = MLX5_ADDR_OF(tirc, tirc,
2038                                              rx_hash_toeplitz_key);
2039                 size_t len = MLX5_FLD_SZ_BYTES(tirc,
2040                                                rx_hash_toeplitz_key);
2041
2042                 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
2043                 memcpy(rss_key, priv->params.toeplitz_hash_key, len);
2044         }
2045 }
2046
2047 static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv)
2048 {
2049         struct mlx5_core_dev *mdev = priv->mdev;
2050
2051         void *in;
2052         void *tirc;
2053         int inlen;
2054         int err;
2055         int tt;
2056         int ix;
2057
2058         inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
2059         in = mlx5_vzalloc(inlen);
2060         if (!in)
2061                 return -ENOMEM;
2062
2063         MLX5_SET(modify_tir_in, in, bitmask.lro, 1);
2064         tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
2065
2066         mlx5e_build_tir_ctx_lro(tirc, priv);
2067
2068         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2069                 err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in,
2070                                            inlen);
2071                 if (err)
2072                         goto free_in;
2073         }
2074
2075         for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
2076                 err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn,
2077                                            in, inlen);
2078                 if (err)
2079                         goto free_in;
2080         }
2081
2082 free_in:
2083         kvfree(in);
2084
2085         return err;
2086 }
2087
2088 static int mlx5e_set_mtu(struct mlx5e_priv *priv, u16 mtu)
2089 {
2090         struct mlx5_core_dev *mdev = priv->mdev;
2091         u16 hw_mtu = MLX5E_SW2HW_MTU(mtu);
2092         int err;
2093
2094         err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
2095         if (err)
2096                 return err;
2097
2098         /* Update vport context MTU */
2099         mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
2100         return 0;
2101 }
2102
2103 static void mlx5e_query_mtu(struct mlx5e_priv *priv, u16 *mtu)
2104 {
2105         struct mlx5_core_dev *mdev = priv->mdev;
2106         u16 hw_mtu = 0;
2107         int err;
2108
2109         err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
2110         if (err || !hw_mtu) /* fallback to port oper mtu */
2111                 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
2112
2113         *mtu = MLX5E_HW2SW_MTU(hw_mtu);
2114 }
2115
2116 static int mlx5e_set_dev_port_mtu(struct net_device *netdev)
2117 {
2118         struct mlx5e_priv *priv = netdev_priv(netdev);
2119         u16 mtu;
2120         int err;
2121
2122         err = mlx5e_set_mtu(priv, netdev->mtu);
2123         if (err)
2124                 return err;
2125
2126         mlx5e_query_mtu(priv, &mtu);
2127         if (mtu != netdev->mtu)
2128                 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
2129                             __func__, mtu, netdev->mtu);
2130
2131         netdev->mtu = mtu;
2132         return 0;
2133 }
2134
2135 static void mlx5e_netdev_set_tcs(struct net_device *netdev)
2136 {
2137         struct mlx5e_priv *priv = netdev_priv(netdev);
2138         int nch = priv->params.num_channels;
2139         int ntc = priv->params.num_tc;
2140         int tc;
2141
2142         netdev_reset_tc(netdev);
2143
2144         if (ntc == 1)
2145                 return;
2146
2147         netdev_set_num_tc(netdev, ntc);
2148
2149         /* Map netdev TCs to offset 0
2150          * We have our own UP to TXQ mapping for QoS
2151          */
2152         for (tc = 0; tc < ntc; tc++)
2153                 netdev_set_tc_queue(netdev, tc, nch, 0);
2154 }
2155
2156 int mlx5e_open_locked(struct net_device *netdev)
2157 {
2158         struct mlx5e_priv *priv = netdev_priv(netdev);
2159         struct mlx5_core_dev *mdev = priv->mdev;
2160         int num_txqs;
2161         int err;
2162
2163         set_bit(MLX5E_STATE_OPENED, &priv->state);
2164
2165         mlx5e_netdev_set_tcs(netdev);
2166
2167         num_txqs = priv->params.num_channels * priv->params.num_tc;
2168         netif_set_real_num_tx_queues(netdev, num_txqs);
2169         netif_set_real_num_rx_queues(netdev, priv->params.num_channels);
2170
2171         err = mlx5e_open_channels(priv);
2172         if (err) {
2173                 netdev_err(netdev, "%s: mlx5e_open_channels failed, %d\n",
2174                            __func__, err);
2175                 goto err_clear_state_opened_flag;
2176         }
2177
2178         err = mlx5e_refresh_tirs_self_loopback(priv->mdev, false);
2179         if (err) {
2180                 netdev_err(netdev, "%s: mlx5e_refresh_tirs_self_loopback_enable failed, %d\n",
2181                            __func__, err);
2182                 goto err_close_channels;
2183         }
2184
2185         mlx5e_redirect_rqts(priv);
2186         mlx5e_update_carrier(priv);
2187         mlx5e_timestamp_init(priv);
2188 #ifdef CONFIG_RFS_ACCEL
2189         priv->netdev->rx_cpu_rmap = priv->mdev->rmap;
2190 #endif
2191         if (priv->profile->update_stats)
2192                 queue_delayed_work(priv->wq, &priv->update_stats_work, 0);
2193
2194         if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
2195                 err = mlx5e_add_sqs_fwd_rules(priv);
2196                 if (err)
2197                         goto err_close_channels;
2198         }
2199         return 0;
2200
2201 err_close_channels:
2202         mlx5e_close_channels(priv);
2203 err_clear_state_opened_flag:
2204         clear_bit(MLX5E_STATE_OPENED, &priv->state);
2205         return err;
2206 }
2207
2208 int mlx5e_open(struct net_device *netdev)
2209 {
2210         struct mlx5e_priv *priv = netdev_priv(netdev);
2211         int err;
2212
2213         mutex_lock(&priv->state_lock);
2214         err = mlx5e_open_locked(netdev);
2215         mutex_unlock(&priv->state_lock);
2216
2217         return err;
2218 }
2219
2220 int mlx5e_close_locked(struct net_device *netdev)
2221 {
2222         struct mlx5e_priv *priv = netdev_priv(netdev);
2223         struct mlx5_core_dev *mdev = priv->mdev;
2224
2225         /* May already be CLOSED in case a previous configuration operation
2226          * (e.g RX/TX queue size change) that involves close&open failed.
2227          */
2228         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2229                 return 0;
2230
2231         clear_bit(MLX5E_STATE_OPENED, &priv->state);
2232
2233         if (MLX5_CAP_GEN(mdev, vport_group_manager))
2234                 mlx5e_remove_sqs_fwd_rules(priv);
2235
2236         mlx5e_timestamp_cleanup(priv);
2237         netif_carrier_off(priv->netdev);
2238         mlx5e_redirect_rqts(priv);
2239         mlx5e_close_channels(priv);
2240
2241         return 0;
2242 }
2243
2244 int mlx5e_close(struct net_device *netdev)
2245 {
2246         struct mlx5e_priv *priv = netdev_priv(netdev);
2247         int err;
2248
2249         if (!netif_device_present(netdev))
2250                 return -ENODEV;
2251
2252         mutex_lock(&priv->state_lock);
2253         err = mlx5e_close_locked(netdev);
2254         mutex_unlock(&priv->state_lock);
2255
2256         return err;
2257 }
2258
2259 static int mlx5e_create_drop_rq(struct mlx5e_priv *priv,
2260                                 struct mlx5e_rq *rq,
2261                                 struct mlx5e_rq_param *param)
2262 {
2263         struct mlx5_core_dev *mdev = priv->mdev;
2264         void *rqc = param->rqc;
2265         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
2266         int err;
2267
2268         param->wq.db_numa_node = param->wq.buf_numa_node;
2269
2270         err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
2271                                 &rq->wq_ctrl);
2272         if (err)
2273                 return err;
2274
2275         rq->priv = priv;
2276
2277         return 0;
2278 }
2279
2280 static int mlx5e_create_drop_cq(struct mlx5e_priv *priv,
2281                                 struct mlx5e_cq *cq,
2282                                 struct mlx5e_cq_param *param)
2283 {
2284         struct mlx5_core_dev *mdev = priv->mdev;
2285         struct mlx5_core_cq *mcq = &cq->mcq;
2286         int eqn_not_used;
2287         unsigned int irqn;
2288         int err;
2289
2290         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
2291                                &cq->wq_ctrl);
2292         if (err)
2293                 return err;
2294
2295         mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
2296
2297         mcq->cqe_sz     = 64;
2298         mcq->set_ci_db  = cq->wq_ctrl.db.db;
2299         mcq->arm_db     = cq->wq_ctrl.db.db + 1;
2300         *mcq->set_ci_db = 0;
2301         *mcq->arm_db    = 0;
2302         mcq->vector     = param->eq_ix;
2303         mcq->comp       = mlx5e_completion_event;
2304         mcq->event      = mlx5e_cq_error_event;
2305         mcq->irqn       = irqn;
2306
2307         cq->priv = priv;
2308
2309         return 0;
2310 }
2311
2312 static int mlx5e_open_drop_rq(struct mlx5e_priv *priv)
2313 {
2314         struct mlx5e_cq_param cq_param;
2315         struct mlx5e_rq_param rq_param;
2316         struct mlx5e_rq *rq = &priv->drop_rq;
2317         struct mlx5e_cq *cq = &priv->drop_rq.cq;
2318         int err;
2319
2320         memset(&cq_param, 0, sizeof(cq_param));
2321         memset(&rq_param, 0, sizeof(rq_param));
2322         mlx5e_build_drop_rq_param(&rq_param);
2323
2324         err = mlx5e_create_drop_cq(priv, cq, &cq_param);
2325         if (err)
2326                 return err;
2327
2328         err = mlx5e_enable_cq(cq, &cq_param);
2329         if (err)
2330                 goto err_destroy_cq;
2331
2332         err = mlx5e_create_drop_rq(priv, rq, &rq_param);
2333         if (err)
2334                 goto err_disable_cq;
2335
2336         err = mlx5e_enable_rq(rq, &rq_param);
2337         if (err)
2338                 goto err_destroy_rq;
2339
2340         return 0;
2341
2342 err_destroy_rq:
2343         mlx5e_destroy_rq(&priv->drop_rq);
2344
2345 err_disable_cq:
2346         mlx5e_disable_cq(&priv->drop_rq.cq);
2347
2348 err_destroy_cq:
2349         mlx5e_destroy_cq(&priv->drop_rq.cq);
2350
2351         return err;
2352 }
2353
2354 static void mlx5e_close_drop_rq(struct mlx5e_priv *priv)
2355 {
2356         mlx5e_disable_rq(&priv->drop_rq);
2357         mlx5e_destroy_rq(&priv->drop_rq);
2358         mlx5e_disable_cq(&priv->drop_rq.cq);
2359         mlx5e_destroy_cq(&priv->drop_rq.cq);
2360 }
2361
2362 static int mlx5e_create_tis(struct mlx5e_priv *priv, int tc)
2363 {
2364         struct mlx5_core_dev *mdev = priv->mdev;
2365         u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
2366         void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
2367
2368         MLX5_SET(tisc, tisc, prio, tc << 1);
2369         MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.td.tdn);
2370
2371         if (mlx5_lag_is_lacp_owner(mdev))
2372                 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
2373
2374         return mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]);
2375 }
2376
2377 static void mlx5e_destroy_tis(struct mlx5e_priv *priv, int tc)
2378 {
2379         mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]);
2380 }
2381
2382 int mlx5e_create_tises(struct mlx5e_priv *priv)
2383 {
2384         int err;
2385         int tc;
2386
2387         for (tc = 0; tc < priv->profile->max_tc; tc++) {
2388                 err = mlx5e_create_tis(priv, tc);
2389                 if (err)
2390                         goto err_close_tises;
2391         }
2392
2393         return 0;
2394
2395 err_close_tises:
2396         for (tc--; tc >= 0; tc--)
2397                 mlx5e_destroy_tis(priv, tc);
2398
2399         return err;
2400 }
2401
2402 void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
2403 {
2404         int tc;
2405
2406         for (tc = 0; tc < priv->profile->max_tc; tc++)
2407                 mlx5e_destroy_tis(priv, tc);
2408 }
2409
2410 static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
2411                                       enum mlx5e_traffic_types tt)
2412 {
2413         void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2414
2415         MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
2416
2417 #define MLX5_HASH_IP            (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2418                                  MLX5_HASH_FIELD_SEL_DST_IP)
2419
2420 #define MLX5_HASH_IP_L4PORTS    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2421                                  MLX5_HASH_FIELD_SEL_DST_IP   |\
2422                                  MLX5_HASH_FIELD_SEL_L4_SPORT |\
2423                                  MLX5_HASH_FIELD_SEL_L4_DPORT)
2424
2425 #define MLX5_HASH_IP_IPSEC_SPI  (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2426                                  MLX5_HASH_FIELD_SEL_DST_IP   |\
2427                                  MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2428
2429         mlx5e_build_tir_ctx_lro(tirc, priv);
2430
2431         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2432         MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn);
2433         mlx5e_build_tir_ctx_hash(tirc, priv);
2434
2435         switch (tt) {
2436         case MLX5E_TT_IPV4_TCP:
2437                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2438                          MLX5_L3_PROT_TYPE_IPV4);
2439                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2440                          MLX5_L4_PROT_TYPE_TCP);
2441                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2442                          MLX5_HASH_IP_L4PORTS);
2443                 break;
2444
2445         case MLX5E_TT_IPV6_TCP:
2446                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2447                          MLX5_L3_PROT_TYPE_IPV6);
2448                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2449                          MLX5_L4_PROT_TYPE_TCP);
2450                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2451                          MLX5_HASH_IP_L4PORTS);
2452                 break;
2453
2454         case MLX5E_TT_IPV4_UDP:
2455                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2456                          MLX5_L3_PROT_TYPE_IPV4);
2457                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2458                          MLX5_L4_PROT_TYPE_UDP);
2459                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2460                          MLX5_HASH_IP_L4PORTS);
2461                 break;
2462
2463         case MLX5E_TT_IPV6_UDP:
2464                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2465                          MLX5_L3_PROT_TYPE_IPV6);
2466                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2467                          MLX5_L4_PROT_TYPE_UDP);
2468                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2469                          MLX5_HASH_IP_L4PORTS);
2470                 break;
2471
2472         case MLX5E_TT_IPV4_IPSEC_AH:
2473                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2474                          MLX5_L3_PROT_TYPE_IPV4);
2475                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2476                          MLX5_HASH_IP_IPSEC_SPI);
2477                 break;
2478
2479         case MLX5E_TT_IPV6_IPSEC_AH:
2480                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2481                          MLX5_L3_PROT_TYPE_IPV6);
2482                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2483                          MLX5_HASH_IP_IPSEC_SPI);
2484                 break;
2485
2486         case MLX5E_TT_IPV4_IPSEC_ESP:
2487                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2488                          MLX5_L3_PROT_TYPE_IPV4);
2489                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2490                          MLX5_HASH_IP_IPSEC_SPI);
2491                 break;
2492
2493         case MLX5E_TT_IPV6_IPSEC_ESP:
2494                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2495                          MLX5_L3_PROT_TYPE_IPV6);
2496                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2497                          MLX5_HASH_IP_IPSEC_SPI);
2498                 break;
2499
2500         case MLX5E_TT_IPV4:
2501                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2502                          MLX5_L3_PROT_TYPE_IPV4);
2503                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2504                          MLX5_HASH_IP);
2505                 break;
2506
2507         case MLX5E_TT_IPV6:
2508                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2509                          MLX5_L3_PROT_TYPE_IPV6);
2510                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2511                          MLX5_HASH_IP);
2512                 break;
2513         default:
2514                 WARN_ONCE(true,
2515                           "mlx5e_build_indir_tir_ctx: bad traffic type!\n");
2516         }
2517 }
2518
2519 static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 *tirc,
2520                                        u32 rqtn)
2521 {
2522         MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
2523
2524         mlx5e_build_tir_ctx_lro(tirc, priv);
2525
2526         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2527         MLX5_SET(tirc, tirc, indirect_table, rqtn);
2528         MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
2529 }
2530
2531 static int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv)
2532 {
2533         struct mlx5e_tir *tir;
2534         void *tirc;
2535         int inlen;
2536         int err;
2537         u32 *in;
2538         int tt;
2539
2540         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2541         in = mlx5_vzalloc(inlen);
2542         if (!in)
2543                 return -ENOMEM;
2544
2545         for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2546                 memset(in, 0, inlen);
2547                 tir = &priv->indir_tir[tt];
2548                 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
2549                 mlx5e_build_indir_tir_ctx(priv, tirc, tt);
2550                 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
2551                 if (err)
2552                         goto err_destroy_tirs;
2553         }
2554
2555         kvfree(in);
2556
2557         return 0;
2558
2559 err_destroy_tirs:
2560         for (tt--; tt >= 0; tt--)
2561                 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]);
2562
2563         kvfree(in);
2564
2565         return err;
2566 }
2567
2568 int mlx5e_create_direct_tirs(struct mlx5e_priv *priv)
2569 {
2570         int nch = priv->profile->max_nch(priv->mdev);
2571         struct mlx5e_tir *tir;
2572         void *tirc;
2573         int inlen;
2574         int err;
2575         u32 *in;
2576         int ix;
2577
2578         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2579         in = mlx5_vzalloc(inlen);
2580         if (!in)
2581                 return -ENOMEM;
2582
2583         for (ix = 0; ix < nch; ix++) {
2584                 memset(in, 0, inlen);
2585                 tir = &priv->direct_tir[ix];
2586                 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
2587                 mlx5e_build_direct_tir_ctx(priv, tirc,
2588                                            priv->direct_tir[ix].rqt.rqtn);
2589                 err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
2590                 if (err)
2591                         goto err_destroy_ch_tirs;
2592         }
2593
2594         kvfree(in);
2595
2596         return 0;
2597
2598 err_destroy_ch_tirs:
2599         for (ix--; ix >= 0; ix--)
2600                 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[ix]);
2601
2602         kvfree(in);
2603
2604         return err;
2605 }
2606
2607 static void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv)
2608 {
2609         int i;
2610
2611         for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
2612                 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]);
2613 }
2614
2615 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv)
2616 {
2617         int nch = priv->profile->max_nch(priv->mdev);
2618         int i;
2619
2620         for (i = 0; i < nch; i++)
2621                 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[i]);
2622 }
2623
2624 int mlx5e_modify_rqs_vsd(struct mlx5e_priv *priv, bool vsd)
2625 {
2626         int err = 0;
2627         int i;
2628
2629         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2630                 return 0;
2631
2632         for (i = 0; i < priv->params.num_channels; i++) {
2633                 err = mlx5e_modify_rq_vsd(&priv->channel[i]->rq, vsd);
2634                 if (err)
2635                         return err;
2636         }
2637
2638         return 0;
2639 }
2640
2641 static int mlx5e_setup_tc(struct net_device *netdev, u8 tc)
2642 {
2643         struct mlx5e_priv *priv = netdev_priv(netdev);
2644         bool was_opened;
2645         int err = 0;
2646
2647         if (tc && tc != MLX5E_MAX_NUM_TC)
2648                 return -EINVAL;
2649
2650         mutex_lock(&priv->state_lock);
2651
2652         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2653         if (was_opened)
2654                 mlx5e_close_locked(priv->netdev);
2655
2656         priv->params.num_tc = tc ? tc : 1;
2657
2658         if (was_opened)
2659                 err = mlx5e_open_locked(priv->netdev);
2660
2661         mutex_unlock(&priv->state_lock);
2662
2663         return err;
2664 }
2665
2666 static int mlx5e_ndo_setup_tc(struct net_device *dev, u32 handle,
2667                               __be16 proto, struct tc_to_netdev *tc)
2668 {
2669         struct mlx5e_priv *priv = netdev_priv(dev);
2670
2671         if (TC_H_MAJ(handle) != TC_H_MAJ(TC_H_INGRESS))
2672                 goto mqprio;
2673
2674         switch (tc->type) {
2675         case TC_SETUP_CLSFLOWER:
2676                 switch (tc->cls_flower->command) {
2677                 case TC_CLSFLOWER_REPLACE:
2678                         return mlx5e_configure_flower(priv, proto, tc->cls_flower);
2679                 case TC_CLSFLOWER_DESTROY:
2680                         return mlx5e_delete_flower(priv, tc->cls_flower);
2681                 case TC_CLSFLOWER_STATS:
2682                         return mlx5e_stats_flower(priv, tc->cls_flower);
2683                 }
2684         default:
2685                 return -EOPNOTSUPP;
2686         }
2687
2688 mqprio:
2689         if (tc->type != TC_SETUP_MQPRIO)
2690                 return -EINVAL;
2691
2692         return mlx5e_setup_tc(dev, tc->tc);
2693 }
2694
2695 static void
2696 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
2697 {
2698         struct mlx5e_priv *priv = netdev_priv(dev);
2699         struct mlx5e_sw_stats *sstats = &priv->stats.sw;
2700         struct mlx5e_vport_stats *vstats = &priv->stats.vport;
2701         struct mlx5e_pport_stats *pstats = &priv->stats.pport;
2702
2703         if (mlx5e_is_uplink_rep(priv)) {
2704                 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok);
2705                 stats->rx_bytes   = PPORT_802_3_GET(pstats, a_octets_received_ok);
2706                 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok);
2707                 stats->tx_bytes   = PPORT_802_3_GET(pstats, a_octets_transmitted_ok);
2708         } else {
2709                 stats->rx_packets = sstats->rx_packets;
2710                 stats->rx_bytes   = sstats->rx_bytes;
2711                 stats->tx_packets = sstats->tx_packets;
2712                 stats->tx_bytes   = sstats->tx_bytes;
2713                 stats->tx_dropped = sstats->tx_queue_dropped;
2714         }
2715
2716         stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
2717
2718         stats->rx_length_errors =
2719                 PPORT_802_3_GET(pstats, a_in_range_length_errors) +
2720                 PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
2721                 PPORT_802_3_GET(pstats, a_frame_too_long_errors);
2722         stats->rx_crc_errors =
2723                 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
2724         stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
2725         stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
2726         stats->tx_carrier_errors =
2727                 PPORT_802_3_GET(pstats, a_symbol_error_during_carrier);
2728         stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
2729                            stats->rx_frame_errors;
2730         stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
2731
2732         /* vport multicast also counts packets that are dropped due to steering
2733          * or rx out of buffer
2734          */
2735         stats->multicast =
2736                 VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
2737
2738 }
2739
2740 static void mlx5e_set_rx_mode(struct net_device *dev)
2741 {
2742         struct mlx5e_priv *priv = netdev_priv(dev);
2743
2744         queue_work(priv->wq, &priv->set_rx_mode_work);
2745 }
2746
2747 static int mlx5e_set_mac(struct net_device *netdev, void *addr)
2748 {
2749         struct mlx5e_priv *priv = netdev_priv(netdev);
2750         struct sockaddr *saddr = addr;
2751
2752         if (!is_valid_ether_addr(saddr->sa_data))
2753                 return -EADDRNOTAVAIL;
2754
2755         netif_addr_lock_bh(netdev);
2756         ether_addr_copy(netdev->dev_addr, saddr->sa_data);
2757         netif_addr_unlock_bh(netdev);
2758
2759         queue_work(priv->wq, &priv->set_rx_mode_work);
2760
2761         return 0;
2762 }
2763
2764 #define MLX5E_SET_FEATURE(netdev, feature, enable)      \
2765         do {                                            \
2766                 if (enable)                             \
2767                         netdev->features |= feature;    \
2768                 else                                    \
2769                         netdev->features &= ~feature;   \
2770         } while (0)
2771
2772 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
2773
2774 static int set_feature_lro(struct net_device *netdev, bool enable)
2775 {
2776         struct mlx5e_priv *priv = netdev_priv(netdev);
2777         bool was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2778         int err;
2779
2780         mutex_lock(&priv->state_lock);
2781
2782         if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
2783                 mlx5e_close_locked(priv->netdev);
2784
2785         priv->params.lro_en = enable;
2786         err = mlx5e_modify_tirs_lro(priv);
2787         if (err) {
2788                 netdev_err(netdev, "lro modify failed, %d\n", err);
2789                 priv->params.lro_en = !enable;
2790         }
2791
2792         if (was_opened && (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST))
2793                 mlx5e_open_locked(priv->netdev);
2794
2795         mutex_unlock(&priv->state_lock);
2796
2797         return err;
2798 }
2799
2800 static int set_feature_vlan_filter(struct net_device *netdev, bool enable)
2801 {
2802         struct mlx5e_priv *priv = netdev_priv(netdev);
2803
2804         if (enable)
2805                 mlx5e_enable_vlan_filter(priv);
2806         else
2807                 mlx5e_disable_vlan_filter(priv);
2808
2809         return 0;
2810 }
2811
2812 static int set_feature_tc_num_filters(struct net_device *netdev, bool enable)
2813 {
2814         struct mlx5e_priv *priv = netdev_priv(netdev);
2815
2816         if (!enable && mlx5e_tc_num_filters(priv)) {
2817                 netdev_err(netdev,
2818                            "Active offloaded tc filters, can't turn hw_tc_offload off\n");
2819                 return -EINVAL;
2820         }
2821
2822         return 0;
2823 }
2824
2825 static int set_feature_rx_all(struct net_device *netdev, bool enable)
2826 {
2827         struct mlx5e_priv *priv = netdev_priv(netdev);
2828         struct mlx5_core_dev *mdev = priv->mdev;
2829
2830         return mlx5_set_port_fcs(mdev, !enable);
2831 }
2832
2833 static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
2834 {
2835         struct mlx5e_priv *priv = netdev_priv(netdev);
2836         int err;
2837
2838         mutex_lock(&priv->state_lock);
2839
2840         priv->params.vlan_strip_disable = !enable;
2841         err = mlx5e_modify_rqs_vsd(priv, !enable);
2842         if (err)
2843                 priv->params.vlan_strip_disable = enable;
2844
2845         mutex_unlock(&priv->state_lock);
2846
2847         return err;
2848 }
2849
2850 #ifdef CONFIG_RFS_ACCEL
2851 static int set_feature_arfs(struct net_device *netdev, bool enable)
2852 {
2853         struct mlx5e_priv *priv = netdev_priv(netdev);
2854         int err;
2855
2856         if (enable)
2857                 err = mlx5e_arfs_enable(priv);
2858         else
2859                 err = mlx5e_arfs_disable(priv);
2860
2861         return err;
2862 }
2863 #endif
2864
2865 static int mlx5e_handle_feature(struct net_device *netdev,
2866                                 netdev_features_t wanted_features,
2867                                 netdev_features_t feature,
2868                                 mlx5e_feature_handler feature_handler)
2869 {
2870         netdev_features_t changes = wanted_features ^ netdev->features;
2871         bool enable = !!(wanted_features & feature);
2872         int err;
2873
2874         if (!(changes & feature))
2875                 return 0;
2876
2877         err = feature_handler(netdev, enable);
2878         if (err) {
2879                 netdev_err(netdev, "%s feature 0x%llx failed err %d\n",
2880                            enable ? "Enable" : "Disable", feature, err);
2881                 return err;
2882         }
2883
2884         MLX5E_SET_FEATURE(netdev, feature, enable);
2885         return 0;
2886 }
2887
2888 static int mlx5e_set_features(struct net_device *netdev,
2889                               netdev_features_t features)
2890 {
2891         int err;
2892
2893         err  = mlx5e_handle_feature(netdev, features, NETIF_F_LRO,
2894                                     set_feature_lro);
2895         err |= mlx5e_handle_feature(netdev, features,
2896                                     NETIF_F_HW_VLAN_CTAG_FILTER,
2897                                     set_feature_vlan_filter);
2898         err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_TC,
2899                                     set_feature_tc_num_filters);
2900         err |= mlx5e_handle_feature(netdev, features, NETIF_F_RXALL,
2901                                     set_feature_rx_all);
2902         err |= mlx5e_handle_feature(netdev, features, NETIF_F_HW_VLAN_CTAG_RX,
2903                                     set_feature_rx_vlan);
2904 #ifdef CONFIG_RFS_ACCEL
2905         err |= mlx5e_handle_feature(netdev, features, NETIF_F_NTUPLE,
2906                                     set_feature_arfs);
2907 #endif
2908
2909         return err ? -EINVAL : 0;
2910 }
2911
2912 static int mlx5e_change_mtu(struct net_device *netdev, int new_mtu)
2913 {
2914         struct mlx5e_priv *priv = netdev_priv(netdev);
2915         bool was_opened;
2916         int err = 0;
2917         bool reset;
2918
2919         mutex_lock(&priv->state_lock);
2920
2921         reset = !priv->params.lro_en &&
2922                 (priv->params.rq_wq_type !=
2923                  MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
2924
2925         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2926         if (was_opened && reset)
2927                 mlx5e_close_locked(netdev);
2928
2929         netdev->mtu = new_mtu;
2930         mlx5e_set_dev_port_mtu(netdev);
2931
2932         if (was_opened && reset)
2933                 err = mlx5e_open_locked(netdev);
2934
2935         mutex_unlock(&priv->state_lock);
2936
2937         return err;
2938 }
2939
2940 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
2941 {
2942         switch (cmd) {
2943         case SIOCSHWTSTAMP:
2944                 return mlx5e_hwstamp_set(dev, ifr);
2945         case SIOCGHWTSTAMP:
2946                 return mlx5e_hwstamp_get(dev, ifr);
2947         default:
2948                 return -EOPNOTSUPP;
2949         }
2950 }
2951
2952 static int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
2953 {
2954         struct mlx5e_priv *priv = netdev_priv(dev);
2955         struct mlx5_core_dev *mdev = priv->mdev;
2956
2957         return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
2958 }
2959
2960 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
2961                              __be16 vlan_proto)
2962 {
2963         struct mlx5e_priv *priv = netdev_priv(dev);
2964         struct mlx5_core_dev *mdev = priv->mdev;
2965
2966         if (vlan_proto != htons(ETH_P_8021Q))
2967                 return -EPROTONOSUPPORT;
2968
2969         return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
2970                                            vlan, qos);
2971 }
2972
2973 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
2974 {
2975         struct mlx5e_priv *priv = netdev_priv(dev);
2976         struct mlx5_core_dev *mdev = priv->mdev;
2977
2978         return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
2979 }
2980
2981 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
2982 {
2983         struct mlx5e_priv *priv = netdev_priv(dev);
2984         struct mlx5_core_dev *mdev = priv->mdev;
2985
2986         return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
2987 }
2988
2989 static int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
2990                              int max_tx_rate)
2991 {
2992         struct mlx5e_priv *priv = netdev_priv(dev);
2993         struct mlx5_core_dev *mdev = priv->mdev;
2994
2995         if (min_tx_rate)
2996                 return -EOPNOTSUPP;
2997
2998         return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1,
2999                                            max_tx_rate);
3000 }
3001
3002 static int mlx5_vport_link2ifla(u8 esw_link)
3003 {
3004         switch (esw_link) {
3005         case MLX5_ESW_VPORT_ADMIN_STATE_DOWN:
3006                 return IFLA_VF_LINK_STATE_DISABLE;
3007         case MLX5_ESW_VPORT_ADMIN_STATE_UP:
3008                 return IFLA_VF_LINK_STATE_ENABLE;
3009         }
3010         return IFLA_VF_LINK_STATE_AUTO;
3011 }
3012
3013 static int mlx5_ifla_link2vport(u8 ifla_link)
3014 {
3015         switch (ifla_link) {
3016         case IFLA_VF_LINK_STATE_DISABLE:
3017                 return MLX5_ESW_VPORT_ADMIN_STATE_DOWN;
3018         case IFLA_VF_LINK_STATE_ENABLE:
3019                 return MLX5_ESW_VPORT_ADMIN_STATE_UP;
3020         }
3021         return MLX5_ESW_VPORT_ADMIN_STATE_AUTO;
3022 }
3023
3024 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
3025                                    int link_state)
3026 {
3027         struct mlx5e_priv *priv = netdev_priv(dev);
3028         struct mlx5_core_dev *mdev = priv->mdev;
3029
3030         return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
3031                                             mlx5_ifla_link2vport(link_state));
3032 }
3033
3034 static int mlx5e_get_vf_config(struct net_device *dev,
3035                                int vf, struct ifla_vf_info *ivi)
3036 {
3037         struct mlx5e_priv *priv = netdev_priv(dev);
3038         struct mlx5_core_dev *mdev = priv->mdev;
3039         int err;
3040
3041         err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
3042         if (err)
3043                 return err;
3044         ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
3045         return 0;
3046 }
3047
3048 static int mlx5e_get_vf_stats(struct net_device *dev,
3049                               int vf, struct ifla_vf_stats *vf_stats)
3050 {
3051         struct mlx5e_priv *priv = netdev_priv(dev);
3052         struct mlx5_core_dev *mdev = priv->mdev;
3053
3054         return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
3055                                             vf_stats);
3056 }
3057
3058 void mlx5e_add_vxlan_port(struct net_device *netdev,
3059                           struct udp_tunnel_info *ti)
3060 {
3061         struct mlx5e_priv *priv = netdev_priv(netdev);
3062
3063         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3064                 return;
3065
3066         if (!mlx5e_vxlan_allowed(priv->mdev))
3067                 return;
3068
3069         mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 1);
3070 }
3071
3072 void mlx5e_del_vxlan_port(struct net_device *netdev,
3073                           struct udp_tunnel_info *ti)
3074 {
3075         struct mlx5e_priv *priv = netdev_priv(netdev);
3076
3077         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
3078                 return;
3079
3080         if (!mlx5e_vxlan_allowed(priv->mdev))
3081                 return;
3082
3083         mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 0);
3084 }
3085
3086 static netdev_features_t mlx5e_vxlan_features_check(struct mlx5e_priv *priv,
3087                                                     struct sk_buff *skb,
3088                                                     netdev_features_t features)
3089 {
3090         struct udphdr *udph;
3091         u16 proto;
3092         u16 port = 0;
3093
3094         switch (vlan_get_protocol(skb)) {
3095         case htons(ETH_P_IP):
3096                 proto = ip_hdr(skb)->protocol;
3097                 break;
3098         case htons(ETH_P_IPV6):
3099                 proto = ipv6_hdr(skb)->nexthdr;
3100                 break;
3101         default:
3102                 goto out;
3103         }
3104
3105         if (proto == IPPROTO_UDP) {
3106                 udph = udp_hdr(skb);
3107                 port = be16_to_cpu(udph->dest);
3108         }
3109
3110         /* Verify if UDP port is being offloaded by HW */
3111         if (port && mlx5e_vxlan_lookup_port(priv, port))
3112                 return features;
3113
3114 out:
3115         /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
3116         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3117 }
3118
3119 static netdev_features_t mlx5e_features_check(struct sk_buff *skb,
3120                                               struct net_device *netdev,
3121                                               netdev_features_t features)
3122 {
3123         struct mlx5e_priv *priv = netdev_priv(netdev);
3124
3125         features = vlan_features_check(skb, features);
3126         features = vxlan_features_check(skb, features);
3127
3128         /* Validate if the tunneled packet is being offloaded by HW */
3129         if (skb->encapsulation &&
3130             (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
3131                 return mlx5e_vxlan_features_check(priv, skb, features);
3132
3133         return features;
3134 }
3135
3136 static void mlx5e_tx_timeout(struct net_device *dev)
3137 {
3138         struct mlx5e_priv *priv = netdev_priv(dev);
3139         bool sched_work = false;
3140         int i;
3141
3142         netdev_err(dev, "TX timeout detected\n");
3143
3144         for (i = 0; i < priv->params.num_channels * priv->params.num_tc; i++) {
3145                 struct mlx5e_sq *sq = priv->txq_to_sq_map[i];
3146
3147                 if (!netif_xmit_stopped(netdev_get_tx_queue(dev, i)))
3148                         continue;
3149                 sched_work = true;
3150                 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
3151                 netdev_err(dev, "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x\n",
3152                            i, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc);
3153         }
3154
3155         if (sched_work && test_bit(MLX5E_STATE_OPENED, &priv->state))
3156                 schedule_work(&priv->tx_timeout_work);
3157 }
3158
3159 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
3160 {
3161         struct mlx5e_priv *priv = netdev_priv(netdev);
3162         struct bpf_prog *old_prog;
3163         int err = 0;
3164         bool reset, was_opened;
3165         int i;
3166
3167         mutex_lock(&priv->state_lock);
3168
3169         if ((netdev->features & NETIF_F_LRO) && prog) {
3170                 netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n");
3171                 err = -EINVAL;
3172                 goto unlock;
3173         }
3174
3175         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3176         /* no need for full reset when exchanging programs */
3177         reset = (!priv->xdp_prog || !prog);
3178
3179         if (was_opened && reset)
3180                 mlx5e_close_locked(netdev);
3181         if (was_opened && !reset) {
3182                 /* num_channels is invariant here, so we can take the
3183                  * batched reference right upfront.
3184                  */
3185                 prog = bpf_prog_add(prog, priv->params.num_channels);
3186                 if (IS_ERR(prog)) {
3187                         err = PTR_ERR(prog);
3188                         goto unlock;
3189                 }
3190         }
3191
3192         /* exchange programs, extra prog reference we got from caller
3193          * as long as we don't fail from this point onwards.
3194          */
3195         old_prog = xchg(&priv->xdp_prog, prog);
3196         if (old_prog)
3197                 bpf_prog_put(old_prog);
3198
3199         if (reset) /* change RQ type according to priv->xdp_prog */
3200                 mlx5e_set_rq_priv_params(priv);
3201
3202         if (was_opened && reset)
3203                 mlx5e_open_locked(netdev);
3204
3205         if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)
3206                 goto unlock;
3207
3208         /* exchanging programs w/o reset, we update ref counts on behalf
3209          * of the channels RQs here.
3210          */
3211         for (i = 0; i < priv->params.num_channels; i++) {
3212                 struct mlx5e_channel *c = priv->channel[i];
3213
3214                 clear_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
3215                 napi_synchronize(&c->napi);
3216                 /* prevent mlx5e_poll_rx_cq from accessing rq->xdp_prog */
3217
3218                 old_prog = xchg(&c->rq.xdp_prog, prog);
3219
3220                 set_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
3221                 /* napi_schedule in case we have missed anything */
3222                 set_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags);
3223                 napi_schedule(&c->napi);
3224
3225                 if (old_prog)
3226                         bpf_prog_put(old_prog);
3227         }
3228
3229 unlock:
3230         mutex_unlock(&priv->state_lock);
3231         return err;
3232 }
3233
3234 static bool mlx5e_xdp_attached(struct net_device *dev)
3235 {
3236         struct mlx5e_priv *priv = netdev_priv(dev);
3237
3238         return !!priv->xdp_prog;
3239 }
3240
3241 static int mlx5e_xdp(struct net_device *dev, struct netdev_xdp *xdp)
3242 {
3243         switch (xdp->command) {
3244         case XDP_SETUP_PROG:
3245                 return mlx5e_xdp_set(dev, xdp->prog);
3246         case XDP_QUERY_PROG:
3247                 xdp->prog_attached = mlx5e_xdp_attached(dev);
3248                 return 0;
3249         default:
3250                 return -EINVAL;
3251         }
3252 }
3253
3254 #ifdef CONFIG_NET_POLL_CONTROLLER
3255 /* Fake "interrupt" called by netpoll (eg netconsole) to send skbs without
3256  * reenabling interrupts.
3257  */
3258 static void mlx5e_netpoll(struct net_device *dev)
3259 {
3260         struct mlx5e_priv *priv = netdev_priv(dev);
3261         int i;
3262
3263         for (i = 0; i < priv->params.num_channels; i++)
3264                 napi_schedule(&priv->channel[i]->napi);
3265 }
3266 #endif
3267
3268 static const struct net_device_ops mlx5e_netdev_ops_basic = {
3269         .ndo_open                = mlx5e_open,
3270         .ndo_stop                = mlx5e_close,
3271         .ndo_start_xmit          = mlx5e_xmit,
3272         .ndo_setup_tc            = mlx5e_ndo_setup_tc,
3273         .ndo_select_queue        = mlx5e_select_queue,
3274         .ndo_get_stats64         = mlx5e_get_stats,
3275         .ndo_set_rx_mode         = mlx5e_set_rx_mode,
3276         .ndo_set_mac_address     = mlx5e_set_mac,
3277         .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
3278         .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
3279         .ndo_set_features        = mlx5e_set_features,
3280         .ndo_change_mtu          = mlx5e_change_mtu,
3281         .ndo_do_ioctl            = mlx5e_ioctl,
3282         .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
3283 #ifdef CONFIG_RFS_ACCEL
3284         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
3285 #endif
3286         .ndo_tx_timeout          = mlx5e_tx_timeout,
3287         .ndo_xdp                 = mlx5e_xdp,
3288 #ifdef CONFIG_NET_POLL_CONTROLLER
3289         .ndo_poll_controller     = mlx5e_netpoll,
3290 #endif
3291 };
3292
3293 static const struct net_device_ops mlx5e_netdev_ops_sriov = {
3294         .ndo_open                = mlx5e_open,
3295         .ndo_stop                = mlx5e_close,
3296         .ndo_start_xmit          = mlx5e_xmit,
3297         .ndo_setup_tc            = mlx5e_ndo_setup_tc,
3298         .ndo_select_queue        = mlx5e_select_queue,
3299         .ndo_get_stats64         = mlx5e_get_stats,
3300         .ndo_set_rx_mode         = mlx5e_set_rx_mode,
3301         .ndo_set_mac_address     = mlx5e_set_mac,
3302         .ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
3303         .ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
3304         .ndo_set_features        = mlx5e_set_features,
3305         .ndo_change_mtu          = mlx5e_change_mtu,
3306         .ndo_do_ioctl            = mlx5e_ioctl,
3307         .ndo_udp_tunnel_add      = mlx5e_add_vxlan_port,
3308         .ndo_udp_tunnel_del      = mlx5e_del_vxlan_port,
3309         .ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
3310         .ndo_features_check      = mlx5e_features_check,
3311 #ifdef CONFIG_RFS_ACCEL
3312         .ndo_rx_flow_steer       = mlx5e_rx_flow_steer,
3313 #endif
3314         .ndo_set_vf_mac          = mlx5e_set_vf_mac,
3315         .ndo_set_vf_vlan         = mlx5e_set_vf_vlan,
3316         .ndo_set_vf_spoofchk     = mlx5e_set_vf_spoofchk,
3317         .ndo_set_vf_trust        = mlx5e_set_vf_trust,
3318         .ndo_set_vf_rate         = mlx5e_set_vf_rate,
3319         .ndo_get_vf_config       = mlx5e_get_vf_config,
3320         .ndo_set_vf_link_state   = mlx5e_set_vf_link_state,
3321         .ndo_get_vf_stats        = mlx5e_get_vf_stats,
3322         .ndo_tx_timeout          = mlx5e_tx_timeout,
3323         .ndo_xdp                 = mlx5e_xdp,
3324 #ifdef CONFIG_NET_POLL_CONTROLLER
3325         .ndo_poll_controller     = mlx5e_netpoll,
3326 #endif
3327         .ndo_has_offload_stats   = mlx5e_has_offload_stats,
3328         .ndo_get_offload_stats   = mlx5e_get_offload_stats,
3329 };
3330
3331 static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
3332 {
3333         if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
3334                 return -ENOTSUPP;
3335         if (!MLX5_CAP_GEN(mdev, eth_net_offloads) ||
3336             !MLX5_CAP_GEN(mdev, nic_flow_table) ||
3337             !MLX5_CAP_ETH(mdev, csum_cap) ||
3338             !MLX5_CAP_ETH(mdev, max_lso_cap) ||
3339             !MLX5_CAP_ETH(mdev, vlan_cap) ||
3340             !MLX5_CAP_ETH(mdev, rss_ind_tbl_cap) ||
3341             MLX5_CAP_FLOWTABLE(mdev,
3342                                flow_table_properties_nic_receive.max_ft_level)
3343                                < 3) {
3344                 mlx5_core_warn(mdev,
3345                                "Not creating net device, some required device capabilities are missing\n");
3346                 return -ENOTSUPP;
3347         }
3348         if (!MLX5_CAP_ETH(mdev, self_lb_en_modifiable))
3349                 mlx5_core_warn(mdev, "Self loop back prevention is not supported\n");
3350         if (!MLX5_CAP_GEN(mdev, cq_moderation))
3351                 mlx5_core_warn(mdev, "CQ modiration is not supported\n");
3352
3353         return 0;
3354 }
3355
3356 u16 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
3357 {
3358         int bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
3359
3360         return bf_buf_size -
3361                sizeof(struct mlx5e_tx_wqe) +
3362                2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/;
3363 }
3364
3365 void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
3366                                    u32 *indirection_rqt, int len,
3367                                    int num_channels)
3368 {
3369         int node = mdev->priv.numa_node;
3370         int node_num_of_cores;
3371         int i;
3372
3373         if (node == -1)
3374                 node = first_online_node;
3375
3376         node_num_of_cores = cpumask_weight(cpumask_of_node(node));
3377
3378         if (node_num_of_cores)
3379                 num_channels = min_t(int, num_channels, node_num_of_cores);
3380
3381         for (i = 0; i < len; i++)
3382                 indirection_rqt[i] = i % num_channels;
3383 }
3384
3385 static int mlx5e_get_pci_bw(struct mlx5_core_dev *mdev, u32 *pci_bw)
3386 {
3387         enum pcie_link_width width;
3388         enum pci_bus_speed speed;
3389         int err = 0;
3390
3391         err = pcie_get_minimum_link(mdev->pdev, &speed, &width);
3392         if (err)
3393                 return err;
3394
3395         if (speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
3396                 return -EINVAL;
3397
3398         switch (speed) {
3399         case PCIE_SPEED_2_5GT:
3400                 *pci_bw = 2500 * width;
3401                 break;
3402         case PCIE_SPEED_5_0GT:
3403                 *pci_bw = 5000 * width;
3404                 break;
3405         case PCIE_SPEED_8_0GT:
3406                 *pci_bw = 8000 * width;
3407                 break;
3408         default:
3409                 return -EINVAL;
3410         }
3411
3412         return 0;
3413 }
3414
3415 static bool cqe_compress_heuristic(u32 link_speed, u32 pci_bw)
3416 {
3417         return (link_speed && pci_bw &&
3418                 (pci_bw < 40000) && (pci_bw < link_speed));
3419 }
3420
3421 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
3422 {
3423         params->rx_cq_period_mode = cq_period_mode;
3424
3425         params->rx_cq_moderation.pkts =
3426                 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
3427         params->rx_cq_moderation.usec =
3428                         MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
3429
3430         if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
3431                 params->rx_cq_moderation.usec =
3432                         MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
3433 }
3434
3435 static void mlx5e_query_min_inline(struct mlx5_core_dev *mdev,
3436                                    u8 *min_inline_mode)
3437 {
3438         switch (MLX5_CAP_ETH(mdev, wqe_inline_mode)) {
3439         case MLX5_CAP_INLINE_MODE_L2:
3440                 *min_inline_mode = MLX5_INLINE_MODE_L2;
3441                 break;
3442         case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
3443                 mlx5_query_nic_vport_min_inline(mdev, 0, min_inline_mode);
3444                 break;
3445         case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
3446                 *min_inline_mode = MLX5_INLINE_MODE_NONE;
3447                 break;
3448         }
3449 }
3450
3451 u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
3452 {
3453         int i;
3454
3455         /* The supported periods are organized in ascending order */
3456         for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++)
3457                 if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout)
3458                         break;
3459
3460         return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]);
3461 }
3462
3463 static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
3464                                         struct net_device *netdev,
3465                                         const struct mlx5e_profile *profile,
3466                                         void *ppriv)
3467 {
3468         struct mlx5e_priv *priv = netdev_priv(netdev);
3469         u32 link_speed = 0;
3470         u32 pci_bw = 0;
3471         u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
3472                                          MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
3473                                          MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
3474
3475         priv->mdev                         = mdev;
3476         priv->netdev                       = netdev;
3477         priv->params.num_channels          = profile->max_nch(mdev);
3478         priv->profile                      = profile;
3479         priv->ppriv                        = ppriv;
3480
3481         priv->params.lro_timeout =
3482                 mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
3483
3484         priv->params.log_sq_size = MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
3485
3486         /* set CQE compression */
3487         priv->params.rx_cqe_compress_def = false;
3488         if (MLX5_CAP_GEN(mdev, cqe_compression) &&
3489             MLX5_CAP_GEN(mdev, vport_group_manager)) {
3490                 mlx5e_get_max_linkspeed(mdev, &link_speed);
3491                 mlx5e_get_pci_bw(mdev, &pci_bw);
3492                 mlx5_core_dbg(mdev, "Max link speed = %d, PCI BW = %d\n",
3493                               link_speed, pci_bw);
3494                 priv->params.rx_cqe_compress_def =
3495                         cqe_compress_heuristic(link_speed, pci_bw);
3496         }
3497
3498         mlx5e_set_rq_priv_params(priv);
3499         if (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
3500                 priv->params.lro_en = true;
3501
3502         priv->params.rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
3503         mlx5e_set_rx_cq_mode_params(&priv->params, cq_period_mode);
3504
3505         priv->params.tx_cq_moderation.usec =
3506                 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
3507         priv->params.tx_cq_moderation.pkts =
3508                 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
3509         priv->params.tx_max_inline         = mlx5e_get_max_inline_cap(mdev);
3510         mlx5e_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
3511         priv->params.num_tc                = 1;
3512         priv->params.rss_hfunc             = ETH_RSS_HASH_XOR;
3513
3514         netdev_rss_key_fill(priv->params.toeplitz_hash_key,
3515                             sizeof(priv->params.toeplitz_hash_key));
3516
3517         mlx5e_build_default_indir_rqt(mdev, priv->params.indirection_rqt,
3518                                       MLX5E_INDIR_RQT_SIZE, profile->max_nch(mdev));
3519
3520         priv->params.lro_wqe_sz =
3521                 MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ -
3522                 /* Extra room needed for build_skb */
3523                 MLX5_RX_HEADROOM -
3524                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3525
3526         /* Initialize pflags */
3527         MLX5E_SET_PFLAG(priv, MLX5E_PFLAG_RX_CQE_BASED_MODER,
3528                         priv->params.rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
3529         MLX5E_SET_PFLAG(priv, MLX5E_PFLAG_RX_CQE_COMPRESS, priv->params.rx_cqe_compress_def);
3530
3531         mutex_init(&priv->state_lock);
3532
3533         INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
3534         INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3535         INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
3536         INIT_DELAYED_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
3537 }
3538
3539 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
3540 {
3541         struct mlx5e_priv *priv = netdev_priv(netdev);
3542
3543         mlx5_query_nic_vport_mac_address(priv->mdev, 0, netdev->dev_addr);
3544         if (is_zero_ether_addr(netdev->dev_addr) &&
3545             !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
3546                 eth_hw_addr_random(netdev);
3547                 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
3548         }
3549 }
3550