net/mlx5e: Add offloading of NIC TC pedit (header re-write) actions
[sfrench/cifs-2.6.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_tc.c
1 /*
2  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32
33 #include <net/flow_dissector.h>
34 #include <net/sch_generic.h>
35 #include <net/pkt_cls.h>
36 #include <net/tc_act/tc_gact.h>
37 #include <net/tc_act/tc_skbedit.h>
38 #include <linux/mlx5/fs.h>
39 #include <linux/mlx5/device.h>
40 #include <linux/rhashtable.h>
41 #include <net/switchdev.h>
42 #include <net/tc_act/tc_mirred.h>
43 #include <net/tc_act/tc_vlan.h>
44 #include <net/tc_act/tc_tunnel_key.h>
45 #include <net/tc_act/tc_pedit.h>
46 #include <net/vxlan.h>
47 #include "en.h"
48 #include "en_tc.h"
49 #include "eswitch.h"
50 #include "vxlan.h"
51
52 struct mlx5_nic_flow_attr {
53         u32 action;
54         u32 flow_tag;
55         u32 mod_hdr_id;
56 };
57
58 enum {
59         MLX5E_TC_FLOW_ESWITCH   = BIT(0),
60         MLX5E_TC_FLOW_NIC       = BIT(1),
61 };
62
63 struct mlx5e_tc_flow {
64         struct rhash_head       node;
65         u64                     cookie;
66         u8                      flags;
67         struct mlx5_flow_handle *rule;
68         struct list_head        encap; /* flows sharing the same encap */
69         union {
70                 struct mlx5_esw_flow_attr esw_attr[0];
71                 struct mlx5_nic_flow_attr nic_attr[0];
72         };
73 };
74
75 struct mlx5e_tc_flow_parse_attr {
76         struct mlx5_flow_spec spec;
77         int num_mod_hdr_actions;
78         void *mod_hdr_actions;
79 };
80
81 enum {
82         MLX5_HEADER_TYPE_VXLAN = 0x0,
83         MLX5_HEADER_TYPE_NVGRE = 0x1,
84 };
85
86 #define MLX5E_TC_TABLE_NUM_ENTRIES 1024
87 #define MLX5E_TC_TABLE_NUM_GROUPS 4
88
89 static struct mlx5_flow_handle *
90 mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
91                       struct mlx5e_tc_flow_parse_attr *parse_attr,
92                       struct mlx5e_tc_flow *flow)
93 {
94         struct mlx5_nic_flow_attr *attr = flow->nic_attr;
95         struct mlx5_core_dev *dev = priv->mdev;
96         struct mlx5_flow_destination dest = {};
97         struct mlx5_flow_act flow_act = {
98                 .action = attr->action,
99                 .flow_tag = attr->flow_tag,
100                 .encap_id = 0,
101                 .modify_id = attr->mod_hdr_id,
102         };
103         struct mlx5_fc *counter = NULL;
104         struct mlx5_flow_handle *rule;
105         bool table_created = false;
106         int err;
107
108         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
109                 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
110                 dest.ft = priv->fs.vlan.ft.t;
111         } else if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
112                 counter = mlx5_fc_create(dev, true);
113                 if (IS_ERR(counter))
114                         return ERR_CAST(counter);
115
116                 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
117                 dest.counter = counter;
118         }
119
120         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) {
121                 err = mlx5_modify_header_alloc(dev, MLX5_FLOW_NAMESPACE_KERNEL,
122                                                parse_attr->num_mod_hdr_actions,
123                                                parse_attr->mod_hdr_actions,
124                                                &attr->mod_hdr_id);
125                 kfree(parse_attr->mod_hdr_actions);
126                 if (err) {
127                         rule = ERR_PTR(err);
128                         goto err_create_mod_hdr_id;
129                 }
130         }
131
132         if (IS_ERR_OR_NULL(priv->fs.tc.t)) {
133                 priv->fs.tc.t =
134                         mlx5_create_auto_grouped_flow_table(priv->fs.ns,
135                                                             MLX5E_TC_PRIO,
136                                                             MLX5E_TC_TABLE_NUM_ENTRIES,
137                                                             MLX5E_TC_TABLE_NUM_GROUPS,
138                                                             0, 0);
139                 if (IS_ERR(priv->fs.tc.t)) {
140                         netdev_err(priv->netdev,
141                                    "Failed to create tc offload table\n");
142                         rule = ERR_CAST(priv->fs.tc.t);
143                         goto err_create_ft;
144                 }
145
146                 table_created = true;
147         }
148
149         parse_attr->spec.match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
150         rule = mlx5_add_flow_rules(priv->fs.tc.t, &parse_attr->spec,
151                                    &flow_act, &dest, 1);
152
153         if (IS_ERR(rule))
154                 goto err_add_rule;
155
156         return rule;
157
158 err_add_rule:
159         if (table_created) {
160                 mlx5_destroy_flow_table(priv->fs.tc.t);
161                 priv->fs.tc.t = NULL;
162         }
163 err_create_ft:
164         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
165                 mlx5_modify_header_dealloc(priv->mdev,
166                                            attr->mod_hdr_id);
167 err_create_mod_hdr_id:
168         mlx5_fc_destroy(dev, counter);
169
170         return rule;
171 }
172
173 static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv,
174                                   struct mlx5e_tc_flow *flow)
175 {
176         struct mlx5_fc *counter = NULL;
177
178         counter = mlx5_flow_rule_counter(flow->rule);
179         mlx5_del_flow_rules(flow->rule);
180         mlx5_fc_destroy(priv->mdev, counter);
181
182         if (!mlx5e_tc_num_filters(priv) && (priv->fs.tc.t)) {
183                 mlx5_destroy_flow_table(priv->fs.tc.t);
184                 priv->fs.tc.t = NULL;
185         }
186
187         if (flow->nic_attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
188                 mlx5_modify_header_dealloc(priv->mdev,
189                                            flow->nic_attr->mod_hdr_id);
190 }
191
192 static void mlx5e_detach_encap(struct mlx5e_priv *priv,
193                                struct mlx5e_tc_flow *flow);
194
195 static struct mlx5_flow_handle *
196 mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv,
197                       struct mlx5e_tc_flow_parse_attr *parse_attr,
198                       struct mlx5e_tc_flow *flow)
199 {
200         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
201         struct mlx5_esw_flow_attr *attr = flow->esw_attr;
202         struct mlx5_flow_handle *rule;
203         int err;
204
205         err = mlx5_eswitch_add_vlan_action(esw, attr);
206         if (err) {
207                 rule = ERR_PTR(err);
208                 goto err_add_vlan;
209         }
210
211         rule = mlx5_eswitch_add_offloaded_rule(esw, &parse_attr->spec, attr);
212         if (IS_ERR(rule))
213                 goto err_add_rule;
214
215         return rule;
216
217 err_add_rule:
218         mlx5_eswitch_del_vlan_action(esw, attr);
219 err_add_vlan:
220         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_ENCAP)
221                 mlx5e_detach_encap(priv, flow);
222
223         return rule;
224 }
225
226 static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv,
227                                   struct mlx5e_tc_flow *flow)
228 {
229         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
230
231         mlx5_eswitch_del_offloaded_rule(esw, flow->rule, flow->esw_attr);
232
233         mlx5_eswitch_del_vlan_action(esw, flow->esw_attr);
234
235         if (flow->esw_attr->action & MLX5_FLOW_CONTEXT_ACTION_ENCAP)
236                 mlx5e_detach_encap(priv, flow);
237 }
238
239 static void mlx5e_detach_encap(struct mlx5e_priv *priv,
240                                struct mlx5e_tc_flow *flow)
241 {
242         struct list_head *next = flow->encap.next;
243
244         list_del(&flow->encap);
245         if (list_empty(next)) {
246                 struct mlx5_encap_entry *e;
247
248                 e = list_entry(next, struct mlx5_encap_entry, flows);
249                 if (e->n) {
250                         mlx5_encap_dealloc(priv->mdev, e->encap_id);
251                         neigh_release(e->n);
252                 }
253                 hlist_del_rcu(&e->encap_hlist);
254                 kfree(e);
255         }
256 }
257
258 static void mlx5e_tc_del_flow(struct mlx5e_priv *priv,
259                               struct mlx5e_tc_flow *flow)
260 {
261         if (flow->flags & MLX5E_TC_FLOW_ESWITCH)
262                 mlx5e_tc_del_fdb_flow(priv, flow);
263         else
264                 mlx5e_tc_del_nic_flow(priv, flow);
265 }
266
267 static void parse_vxlan_attr(struct mlx5_flow_spec *spec,
268                              struct tc_cls_flower_offload *f)
269 {
270         void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
271                                        outer_headers);
272         void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
273                                        outer_headers);
274         void *misc_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
275                                     misc_parameters);
276         void *misc_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
277                                     misc_parameters);
278
279         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol);
280         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
281
282         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
283                 struct flow_dissector_key_keyid *key =
284                         skb_flow_dissector_target(f->dissector,
285                                                   FLOW_DISSECTOR_KEY_ENC_KEYID,
286                                                   f->key);
287                 struct flow_dissector_key_keyid *mask =
288                         skb_flow_dissector_target(f->dissector,
289                                                   FLOW_DISSECTOR_KEY_ENC_KEYID,
290                                                   f->mask);
291                 MLX5_SET(fte_match_set_misc, misc_c, vxlan_vni,
292                          be32_to_cpu(mask->keyid));
293                 MLX5_SET(fte_match_set_misc, misc_v, vxlan_vni,
294                          be32_to_cpu(key->keyid));
295         }
296 }
297
298 static int parse_tunnel_attr(struct mlx5e_priv *priv,
299                              struct mlx5_flow_spec *spec,
300                              struct tc_cls_flower_offload *f)
301 {
302         void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
303                                        outer_headers);
304         void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
305                                        outer_headers);
306
307         struct flow_dissector_key_control *enc_control =
308                 skb_flow_dissector_target(f->dissector,
309                                           FLOW_DISSECTOR_KEY_ENC_CONTROL,
310                                           f->key);
311
312         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
313                 struct flow_dissector_key_ports *key =
314                         skb_flow_dissector_target(f->dissector,
315                                                   FLOW_DISSECTOR_KEY_ENC_PORTS,
316                                                   f->key);
317                 struct flow_dissector_key_ports *mask =
318                         skb_flow_dissector_target(f->dissector,
319                                                   FLOW_DISSECTOR_KEY_ENC_PORTS,
320                                                   f->mask);
321                 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
322                 struct net_device *up_dev = mlx5_eswitch_get_uplink_netdev(esw);
323                 struct mlx5e_priv *up_priv = netdev_priv(up_dev);
324
325                 /* Full udp dst port must be given */
326                 if (memchr_inv(&mask->dst, 0xff, sizeof(mask->dst)))
327                         goto vxlan_match_offload_err;
328
329                 if (mlx5e_vxlan_lookup_port(up_priv, be16_to_cpu(key->dst)) &&
330                     MLX5_CAP_ESW(priv->mdev, vxlan_encap_decap))
331                         parse_vxlan_attr(spec, f);
332                 else {
333                         netdev_warn(priv->netdev,
334                                     "%d isn't an offloaded vxlan udp dport\n", be16_to_cpu(key->dst));
335                         return -EOPNOTSUPP;
336                 }
337
338                 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
339                          udp_dport, ntohs(mask->dst));
340                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
341                          udp_dport, ntohs(key->dst));
342
343                 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
344                          udp_sport, ntohs(mask->src));
345                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
346                          udp_sport, ntohs(key->src));
347         } else { /* udp dst port must be given */
348 vxlan_match_offload_err:
349                 netdev_warn(priv->netdev,
350                             "IP tunnel decap offload supported only for vxlan, must set UDP dport\n");
351                 return -EOPNOTSUPP;
352         }
353
354         if (enc_control->addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
355                 struct flow_dissector_key_ipv4_addrs *key =
356                         skb_flow_dissector_target(f->dissector,
357                                                   FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
358                                                   f->key);
359                 struct flow_dissector_key_ipv4_addrs *mask =
360                         skb_flow_dissector_target(f->dissector,
361                                                   FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
362                                                   f->mask);
363                 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
364                          src_ipv4_src_ipv6.ipv4_layout.ipv4,
365                          ntohl(mask->src));
366                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
367                          src_ipv4_src_ipv6.ipv4_layout.ipv4,
368                          ntohl(key->src));
369
370                 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
371                          dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
372                          ntohl(mask->dst));
373                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
374                          dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
375                          ntohl(key->dst));
376
377                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ethertype);
378                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, ETH_P_IP);
379         } else if (enc_control->addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
380                 struct flow_dissector_key_ipv6_addrs *key =
381                         skb_flow_dissector_target(f->dissector,
382                                                   FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS,
383                                                   f->key);
384                 struct flow_dissector_key_ipv6_addrs *mask =
385                         skb_flow_dissector_target(f->dissector,
386                                                   FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS,
387                                                   f->mask);
388
389                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
390                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
391                        &mask->src, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6));
392                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
393                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
394                        &key->src, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6));
395
396                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
397                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
398                        &mask->dst, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6));
399                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
400                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
401                        &key->dst, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6));
402
403                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ethertype);
404                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, ETH_P_IPV6);
405         }
406
407         /* Enforce DMAC when offloading incoming tunneled flows.
408          * Flow counters require a match on the DMAC.
409          */
410         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, dmac_47_16);
411         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, dmac_15_0);
412         ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
413                                      dmac_47_16), priv->netdev->dev_addr);
414
415         /* let software handle IP fragments */
416         MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
417         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
418
419         return 0;
420 }
421
422 static int __parse_cls_flower(struct mlx5e_priv *priv,
423                               struct mlx5_flow_spec *spec,
424                               struct tc_cls_flower_offload *f,
425                               u8 *min_inline)
426 {
427         void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
428                                        outer_headers);
429         void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
430                                        outer_headers);
431         u16 addr_type = 0;
432         u8 ip_proto = 0;
433
434         *min_inline = MLX5_INLINE_MODE_L2;
435
436         if (f->dissector->used_keys &
437             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
438               BIT(FLOW_DISSECTOR_KEY_BASIC) |
439               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
440               BIT(FLOW_DISSECTOR_KEY_VLAN) |
441               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
442               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
443               BIT(FLOW_DISSECTOR_KEY_PORTS) |
444               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) |
445               BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
446               BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
447               BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) |
448               BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL))) {
449                 netdev_warn(priv->netdev, "Unsupported key used: 0x%x\n",
450                             f->dissector->used_keys);
451                 return -EOPNOTSUPP;
452         }
453
454         if ((dissector_uses_key(f->dissector,
455                                 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) ||
456              dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID) ||
457              dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) &&
458             dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
459                 struct flow_dissector_key_control *key =
460                         skb_flow_dissector_target(f->dissector,
461                                                   FLOW_DISSECTOR_KEY_ENC_CONTROL,
462                                                   f->key);
463                 switch (key->addr_type) {
464                 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
465                 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
466                         if (parse_tunnel_attr(priv, spec, f))
467                                 return -EOPNOTSUPP;
468                         break;
469                 default:
470                         return -EOPNOTSUPP;
471                 }
472
473                 /* In decap flow, header pointers should point to the inner
474                  * headers, outer header were already set by parse_tunnel_attr
475                  */
476                 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
477                                          inner_headers);
478                 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
479                                          inner_headers);
480         }
481
482         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
483                 struct flow_dissector_key_control *key =
484                         skb_flow_dissector_target(f->dissector,
485                                                   FLOW_DISSECTOR_KEY_CONTROL,
486                                                   f->key);
487
488                 struct flow_dissector_key_control *mask =
489                         skb_flow_dissector_target(f->dissector,
490                                                   FLOW_DISSECTOR_KEY_CONTROL,
491                                                   f->mask);
492                 addr_type = key->addr_type;
493
494                 if (mask->flags & FLOW_DIS_IS_FRAGMENT) {
495                         MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
496                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
497                                  key->flags & FLOW_DIS_IS_FRAGMENT);
498
499                         /* the HW doesn't need L3 inline to match on frag=no */
500                         if (key->flags & FLOW_DIS_IS_FRAGMENT)
501                                 *min_inline = MLX5_INLINE_MODE_IP;
502                 }
503         }
504
505         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
506                 struct flow_dissector_key_basic *key =
507                         skb_flow_dissector_target(f->dissector,
508                                                   FLOW_DISSECTOR_KEY_BASIC,
509                                                   f->key);
510                 struct flow_dissector_key_basic *mask =
511                         skb_flow_dissector_target(f->dissector,
512                                                   FLOW_DISSECTOR_KEY_BASIC,
513                                                   f->mask);
514                 ip_proto = key->ip_proto;
515
516                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype,
517                          ntohs(mask->n_proto));
518                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
519                          ntohs(key->n_proto));
520
521                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
522                          mask->ip_proto);
523                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
524                          key->ip_proto);
525
526                 if (mask->ip_proto)
527                         *min_inline = MLX5_INLINE_MODE_IP;
528         }
529
530         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
531                 struct flow_dissector_key_eth_addrs *key =
532                         skb_flow_dissector_target(f->dissector,
533                                                   FLOW_DISSECTOR_KEY_ETH_ADDRS,
534                                                   f->key);
535                 struct flow_dissector_key_eth_addrs *mask =
536                         skb_flow_dissector_target(f->dissector,
537                                                   FLOW_DISSECTOR_KEY_ETH_ADDRS,
538                                                   f->mask);
539
540                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
541                                              dmac_47_16),
542                                 mask->dst);
543                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
544                                              dmac_47_16),
545                                 key->dst);
546
547                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
548                                              smac_47_16),
549                                 mask->src);
550                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
551                                              smac_47_16),
552                                 key->src);
553         }
554
555         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
556                 struct flow_dissector_key_vlan *key =
557                         skb_flow_dissector_target(f->dissector,
558                                                   FLOW_DISSECTOR_KEY_VLAN,
559                                                   f->key);
560                 struct flow_dissector_key_vlan *mask =
561                         skb_flow_dissector_target(f->dissector,
562                                                   FLOW_DISSECTOR_KEY_VLAN,
563                                                   f->mask);
564                 if (mask->vlan_id || mask->vlan_priority) {
565                         MLX5_SET(fte_match_set_lyr_2_4, headers_c, cvlan_tag, 1);
566                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
567
568                         MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_vid, mask->vlan_id);
569                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, key->vlan_id);
570
571                         MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_prio, mask->vlan_priority);
572                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, key->vlan_priority);
573                 }
574         }
575
576         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
577                 struct flow_dissector_key_ipv4_addrs *key =
578                         skb_flow_dissector_target(f->dissector,
579                                                   FLOW_DISSECTOR_KEY_IPV4_ADDRS,
580                                                   f->key);
581                 struct flow_dissector_key_ipv4_addrs *mask =
582                         skb_flow_dissector_target(f->dissector,
583                                                   FLOW_DISSECTOR_KEY_IPV4_ADDRS,
584                                                   f->mask);
585
586                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
587                                     src_ipv4_src_ipv6.ipv4_layout.ipv4),
588                        &mask->src, sizeof(mask->src));
589                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
590                                     src_ipv4_src_ipv6.ipv4_layout.ipv4),
591                        &key->src, sizeof(key->src));
592                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
593                                     dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
594                        &mask->dst, sizeof(mask->dst));
595                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
596                                     dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
597                        &key->dst, sizeof(key->dst));
598
599                 if (mask->src || mask->dst)
600                         *min_inline = MLX5_INLINE_MODE_IP;
601         }
602
603         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
604                 struct flow_dissector_key_ipv6_addrs *key =
605                         skb_flow_dissector_target(f->dissector,
606                                                   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
607                                                   f->key);
608                 struct flow_dissector_key_ipv6_addrs *mask =
609                         skb_flow_dissector_target(f->dissector,
610                                                   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
611                                                   f->mask);
612
613                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
614                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
615                        &mask->src, sizeof(mask->src));
616                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
617                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
618                        &key->src, sizeof(key->src));
619
620                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
621                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
622                        &mask->dst, sizeof(mask->dst));
623                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
624                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
625                        &key->dst, sizeof(key->dst));
626
627                 if (ipv6_addr_type(&mask->src) != IPV6_ADDR_ANY ||
628                     ipv6_addr_type(&mask->dst) != IPV6_ADDR_ANY)
629                         *min_inline = MLX5_INLINE_MODE_IP;
630         }
631
632         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) {
633                 struct flow_dissector_key_ports *key =
634                         skb_flow_dissector_target(f->dissector,
635                                                   FLOW_DISSECTOR_KEY_PORTS,
636                                                   f->key);
637                 struct flow_dissector_key_ports *mask =
638                         skb_flow_dissector_target(f->dissector,
639                                                   FLOW_DISSECTOR_KEY_PORTS,
640                                                   f->mask);
641                 switch (ip_proto) {
642                 case IPPROTO_TCP:
643                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
644                                  tcp_sport, ntohs(mask->src));
645                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
646                                  tcp_sport, ntohs(key->src));
647
648                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
649                                  tcp_dport, ntohs(mask->dst));
650                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
651                                  tcp_dport, ntohs(key->dst));
652                         break;
653
654                 case IPPROTO_UDP:
655                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
656                                  udp_sport, ntohs(mask->src));
657                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
658                                  udp_sport, ntohs(key->src));
659
660                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
661                                  udp_dport, ntohs(mask->dst));
662                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
663                                  udp_dport, ntohs(key->dst));
664                         break;
665                 default:
666                         netdev_err(priv->netdev,
667                                    "Only UDP and TCP transport are supported\n");
668                         return -EINVAL;
669                 }
670
671                 if (mask->src || mask->dst)
672                         *min_inline = MLX5_INLINE_MODE_TCP_UDP;
673         }
674
675         return 0;
676 }
677
678 static int parse_cls_flower(struct mlx5e_priv *priv,
679                             struct mlx5e_tc_flow *flow,
680                             struct mlx5_flow_spec *spec,
681                             struct tc_cls_flower_offload *f)
682 {
683         struct mlx5_core_dev *dev = priv->mdev;
684         struct mlx5_eswitch *esw = dev->priv.eswitch;
685         struct mlx5_eswitch_rep *rep = priv->ppriv;
686         u8 min_inline;
687         int err;
688
689         err = __parse_cls_flower(priv, spec, f, &min_inline);
690
691         if (!err && (flow->flags & MLX5E_TC_FLOW_ESWITCH) &&
692             rep->vport != FDB_UPLINK_VPORT) {
693                 if (min_inline > esw->offloads.inline_mode) {
694                         netdev_warn(priv->netdev,
695                                     "Flow is not offloaded due to min inline setting, required %d actual %d\n",
696                                     min_inline, esw->offloads.inline_mode);
697                         return -EOPNOTSUPP;
698                 }
699         }
700
701         return err;
702 }
703
704 struct pedit_headers {
705         struct ethhdr  eth;
706         struct iphdr   ip4;
707         struct ipv6hdr ip6;
708         struct tcphdr  tcp;
709         struct udphdr  udp;
710 };
711
712 static int pedit_header_offsets[] = {
713         [TCA_PEDIT_KEY_EX_HDR_TYPE_ETH] = offsetof(struct pedit_headers, eth),
714         [TCA_PEDIT_KEY_EX_HDR_TYPE_IP4] = offsetof(struct pedit_headers, ip4),
715         [TCA_PEDIT_KEY_EX_HDR_TYPE_IP6] = offsetof(struct pedit_headers, ip6),
716         [TCA_PEDIT_KEY_EX_HDR_TYPE_TCP] = offsetof(struct pedit_headers, tcp),
717         [TCA_PEDIT_KEY_EX_HDR_TYPE_UDP] = offsetof(struct pedit_headers, udp),
718 };
719
720 #define pedit_header(_ph, _htype) ((void *)(_ph) + pedit_header_offsets[_htype])
721
722 static int set_pedit_val(u8 hdr_type, u32 mask, u32 val, u32 offset,
723                          struct pedit_headers *masks,
724                          struct pedit_headers *vals)
725 {
726         u32 *curr_pmask, *curr_pval;
727
728         if (hdr_type >= __PEDIT_HDR_TYPE_MAX)
729                 goto out_err;
730
731         curr_pmask = (u32 *)(pedit_header(masks, hdr_type) + offset);
732         curr_pval  = (u32 *)(pedit_header(vals, hdr_type) + offset);
733
734         if (*curr_pmask & mask)  /* disallow acting twice on the same location */
735                 goto out_err;
736
737         *curr_pmask |= mask;
738         *curr_pval  |= (val & mask);
739
740         return 0;
741
742 out_err:
743         return -EOPNOTSUPP;
744 }
745
746 struct mlx5_fields {
747         u8  field;
748         u8  size;
749         u32 offset;
750 };
751
752 static struct mlx5_fields fields[] = {
753         {MLX5_ACTION_IN_FIELD_OUT_DMAC_47_16, 4, offsetof(struct pedit_headers, eth.h_dest[0])},
754         {MLX5_ACTION_IN_FIELD_OUT_DMAC_15_0,  2, offsetof(struct pedit_headers, eth.h_dest[4])},
755         {MLX5_ACTION_IN_FIELD_OUT_SMAC_47_16, 4, offsetof(struct pedit_headers, eth.h_source[0])},
756         {MLX5_ACTION_IN_FIELD_OUT_SMAC_15_0,  2, offsetof(struct pedit_headers, eth.h_source[4])},
757         {MLX5_ACTION_IN_FIELD_OUT_ETHERTYPE,  2, offsetof(struct pedit_headers, eth.h_proto)},
758
759         {MLX5_ACTION_IN_FIELD_OUT_IP_DSCP, 1, offsetof(struct pedit_headers, ip4.tos)},
760         {MLX5_ACTION_IN_FIELD_OUT_IP_TTL,  1, offsetof(struct pedit_headers, ip4.ttl)},
761         {MLX5_ACTION_IN_FIELD_OUT_SIPV4,   4, offsetof(struct pedit_headers, ip4.saddr)},
762         {MLX5_ACTION_IN_FIELD_OUT_DIPV4,   4, offsetof(struct pedit_headers, ip4.daddr)},
763
764         {MLX5_ACTION_IN_FIELD_OUT_SIPV6_127_96, 4, offsetof(struct pedit_headers, ip6.saddr.s6_addr32[0])},
765         {MLX5_ACTION_IN_FIELD_OUT_SIPV6_95_64,  4, offsetof(struct pedit_headers, ip6.saddr.s6_addr32[1])},
766         {MLX5_ACTION_IN_FIELD_OUT_SIPV6_63_32,  4, offsetof(struct pedit_headers, ip6.saddr.s6_addr32[2])},
767         {MLX5_ACTION_IN_FIELD_OUT_SIPV6_31_0,   4, offsetof(struct pedit_headers, ip6.saddr.s6_addr32[3])},
768         {MLX5_ACTION_IN_FIELD_OUT_DIPV6_127_96, 4, offsetof(struct pedit_headers, ip6.daddr.s6_addr32[0])},
769         {MLX5_ACTION_IN_FIELD_OUT_DIPV6_95_64,  4, offsetof(struct pedit_headers, ip6.daddr.s6_addr32[1])},
770         {MLX5_ACTION_IN_FIELD_OUT_DIPV6_63_32,  4, offsetof(struct pedit_headers, ip6.daddr.s6_addr32[2])},
771         {MLX5_ACTION_IN_FIELD_OUT_DIPV6_31_0,   4, offsetof(struct pedit_headers, ip6.daddr.s6_addr32[3])},
772
773         {MLX5_ACTION_IN_FIELD_OUT_TCP_SPORT, 2, offsetof(struct pedit_headers, tcp.source)},
774         {MLX5_ACTION_IN_FIELD_OUT_TCP_DPORT, 2, offsetof(struct pedit_headers, tcp.dest)},
775         {MLX5_ACTION_IN_FIELD_OUT_TCP_FLAGS, 1, offsetof(struct pedit_headers, tcp.ack_seq) + 5},
776
777         {MLX5_ACTION_IN_FIELD_OUT_UDP_SPORT, 2, offsetof(struct pedit_headers, udp.source)},
778         {MLX5_ACTION_IN_FIELD_OUT_UDP_DPORT, 2, offsetof(struct pedit_headers, udp.dest)},
779 };
780
781 /* On input attr->num_mod_hdr_actions tells how many HW actions can be parsed at
782  * max from the SW pedit action. On success, it says how many HW actions were
783  * actually parsed.
784  */
785 static int offload_pedit_fields(struct pedit_headers *masks,
786                                 struct pedit_headers *vals,
787                                 struct mlx5e_tc_flow_parse_attr *parse_attr)
788 {
789         struct pedit_headers *set_masks, *add_masks, *set_vals, *add_vals;
790         int i, action_size, nactions, max_actions, first, last;
791         void *s_masks_p, *a_masks_p, *vals_p;
792         u32 s_mask, a_mask, val;
793         struct mlx5_fields *f;
794         u8 cmd, field_bsize;
795         unsigned long mask;
796         void *action;
797
798         set_masks = &masks[TCA_PEDIT_KEY_EX_CMD_SET];
799         add_masks = &masks[TCA_PEDIT_KEY_EX_CMD_ADD];
800         set_vals = &vals[TCA_PEDIT_KEY_EX_CMD_SET];
801         add_vals = &vals[TCA_PEDIT_KEY_EX_CMD_ADD];
802
803         action_size = MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto);
804         action = parse_attr->mod_hdr_actions;
805         max_actions = parse_attr->num_mod_hdr_actions;
806         nactions = 0;
807
808         for (i = 0; i < ARRAY_SIZE(fields); i++) {
809                 f = &fields[i];
810                 /* avoid seeing bits set from previous iterations */
811                 s_mask = a_mask = mask = val = 0;
812
813                 s_masks_p = (void *)set_masks + f->offset;
814                 a_masks_p = (void *)add_masks + f->offset;
815
816                 memcpy(&s_mask, s_masks_p, f->size);
817                 memcpy(&a_mask, a_masks_p, f->size);
818
819                 if (!s_mask && !a_mask) /* nothing to offload here */
820                         continue;
821
822                 if (s_mask && a_mask) {
823                         printk(KERN_WARNING "mlx5: can't set and add to the same HW field (%x)\n", f->field);
824                         return -EOPNOTSUPP;
825                 }
826
827                 if (nactions == max_actions) {
828                         printk(KERN_WARNING "mlx5: parsed %d pedit actions, can't do more\n", nactions);
829                         return -EOPNOTSUPP;
830                 }
831
832                 if (s_mask) {
833                         cmd  = MLX5_ACTION_TYPE_SET;
834                         mask = s_mask;
835                         vals_p = (void *)set_vals + f->offset;
836                         /* clear to denote we consumed this field */
837                         memset(s_masks_p, 0, f->size);
838                 } else {
839                         cmd  = MLX5_ACTION_TYPE_ADD;
840                         mask = a_mask;
841                         vals_p = (void *)add_vals + f->offset;
842                         /* clear to denote we consumed this field */
843                         memset(a_masks_p, 0, f->size);
844                 }
845
846                 memcpy(&val, vals_p, f->size);
847
848                 field_bsize = f->size * BITS_PER_BYTE;
849                 first = find_first_bit(&mask, field_bsize);
850                 last  = find_last_bit(&mask, field_bsize);
851                 if (first > 0 || last != (field_bsize - 1)) {
852                         printk(KERN_WARNING "mlx5: partial rewrite (mask %lx) is currently not offloaded\n",
853                                mask);
854                         return -EOPNOTSUPP;
855                 }
856
857                 MLX5_SET(set_action_in, action, action_type, cmd);
858                 MLX5_SET(set_action_in, action, field, f->field);
859
860                 if (cmd == MLX5_ACTION_TYPE_SET) {
861                         MLX5_SET(set_action_in, action, offset, 0);
862                         /* length is num of bits to be written, zero means length of 32 */
863                         MLX5_SET(set_action_in, action, length, field_bsize);
864                 }
865
866                 if (field_bsize == 32)
867                         MLX5_SET(set_action_in, action, data, ntohl(val));
868                 else if (field_bsize == 16)
869                         MLX5_SET(set_action_in, action, data, ntohs(val));
870                 else if (field_bsize == 8)
871                         MLX5_SET(set_action_in, action, data, val);
872
873                 action += action_size;
874                 nactions++;
875         }
876
877         parse_attr->num_mod_hdr_actions = nactions;
878         return 0;
879 }
880
881 static int alloc_mod_hdr_actions(struct mlx5e_priv *priv,
882                                  const struct tc_action *a, int namespace,
883                                  struct mlx5e_tc_flow_parse_attr *parse_attr)
884 {
885         int nkeys, action_size, max_actions;
886
887         nkeys = tcf_pedit_nkeys(a);
888         action_size = MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto);
889
890         if (namespace == MLX5_FLOW_NAMESPACE_FDB) /* FDB offloading */
891                 max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(priv->mdev, max_modify_header_actions);
892         else /* namespace is MLX5_FLOW_NAMESPACE_KERNEL - NIC offloading */
893                 max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, max_modify_header_actions);
894
895         /* can get up to crazingly 16 HW actions in 32 bits pedit SW key */
896         max_actions = min(max_actions, nkeys * 16);
897
898         parse_attr->mod_hdr_actions = kcalloc(max_actions, action_size, GFP_KERNEL);
899         if (!parse_attr->mod_hdr_actions)
900                 return -ENOMEM;
901
902         parse_attr->num_mod_hdr_actions = max_actions;
903         return 0;
904 }
905
906 static const struct pedit_headers zero_masks = {};
907
908 static int parse_tc_pedit_action(struct mlx5e_priv *priv,
909                                  const struct tc_action *a, int namespace,
910                                  struct mlx5e_tc_flow_parse_attr *parse_attr)
911 {
912         struct pedit_headers masks[__PEDIT_CMD_MAX], vals[__PEDIT_CMD_MAX], *cmd_masks;
913         int nkeys, i, err = -EOPNOTSUPP;
914         u32 mask, val, offset;
915         u8 cmd, htype;
916
917         nkeys = tcf_pedit_nkeys(a);
918
919         memset(masks, 0, sizeof(struct pedit_headers) * __PEDIT_CMD_MAX);
920         memset(vals,  0, sizeof(struct pedit_headers) * __PEDIT_CMD_MAX);
921
922         for (i = 0; i < nkeys; i++) {
923                 htype = tcf_pedit_htype(a, i);
924                 cmd = tcf_pedit_cmd(a, i);
925                 err = -EOPNOTSUPP; /* can't be all optimistic */
926
927                 if (htype == TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK) {
928                         printk(KERN_WARNING "mlx5: legacy pedit isn't offloaded\n");
929                         goto out_err;
930                 }
931
932                 if (cmd != TCA_PEDIT_KEY_EX_CMD_SET && cmd != TCA_PEDIT_KEY_EX_CMD_ADD) {
933                         printk(KERN_WARNING "mlx5: pedit cmd %d isn't offloaded\n", cmd);
934                         goto out_err;
935                 }
936
937                 mask = tcf_pedit_mask(a, i);
938                 val = tcf_pedit_val(a, i);
939                 offset = tcf_pedit_offset(a, i);
940
941                 err = set_pedit_val(htype, ~mask, val, offset, &masks[cmd], &vals[cmd]);
942                 if (err)
943                         goto out_err;
944         }
945
946         err = alloc_mod_hdr_actions(priv, a, namespace, parse_attr);
947         if (err)
948                 goto out_err;
949
950         err = offload_pedit_fields(masks, vals, parse_attr);
951         if (err < 0)
952                 goto out_dealloc_parsed_actions;
953
954         for (cmd = 0; cmd < __PEDIT_CMD_MAX; cmd++) {
955                 cmd_masks = &masks[cmd];
956                 if (memcmp(cmd_masks, &zero_masks, sizeof(zero_masks))) {
957                         printk(KERN_WARNING "mlx5: attempt to offload an unsupported field (cmd %d)\n",
958                                cmd);
959                         print_hex_dump(KERN_WARNING, "mask: ", DUMP_PREFIX_ADDRESS,
960                                        16, 1, cmd_masks, sizeof(zero_masks), true);
961                         err = -EOPNOTSUPP;
962                         goto out_dealloc_parsed_actions;
963                 }
964         }
965
966         return 0;
967
968 out_dealloc_parsed_actions:
969         kfree(parse_attr->mod_hdr_actions);
970 out_err:
971         return err;
972 }
973
974 static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
975                                 struct mlx5e_tc_flow_parse_attr *parse_attr,
976                                 struct mlx5e_tc_flow *flow)
977 {
978         struct mlx5_nic_flow_attr *attr = flow->nic_attr;
979         const struct tc_action *a;
980         LIST_HEAD(actions);
981         int err;
982
983         if (tc_no_actions(exts))
984                 return -EINVAL;
985
986         attr->flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
987         attr->action = 0;
988
989         tcf_exts_to_list(exts, &actions);
990         list_for_each_entry(a, &actions, list) {
991                 /* Only support a single action per rule */
992                 if (attr->action)
993                         return -EINVAL;
994
995                 if (is_tcf_gact_shot(a)) {
996                         attr->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
997                         if (MLX5_CAP_FLOWTABLE(priv->mdev,
998                                                flow_table_properties_nic_receive.flow_counter))
999                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
1000                         continue;
1001                 }
1002
1003                 if (is_tcf_pedit(a)) {
1004                         err = parse_tc_pedit_action(priv, a, MLX5_FLOW_NAMESPACE_KERNEL,
1005                                                     parse_attr);
1006                         if (err)
1007                                 return err;
1008
1009                         attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR |
1010                                         MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1011                         continue;
1012                 }
1013
1014                 if (is_tcf_skbedit_mark(a)) {
1015                         u32 mark = tcf_skbedit_mark(a);
1016
1017                         if (mark & ~MLX5E_TC_FLOW_ID_MASK) {
1018                                 netdev_warn(priv->netdev, "Bad flow mark - only 16 bit is supported: 0x%x\n",
1019                                             mark);
1020                                 return -EINVAL;
1021                         }
1022
1023                         attr->flow_tag = mark;
1024                         attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1025                         continue;
1026                 }
1027
1028                 return -EINVAL;
1029         }
1030
1031         return 0;
1032 }
1033
1034 static inline int cmp_encap_info(struct ip_tunnel_key *a,
1035                                  struct ip_tunnel_key *b)
1036 {
1037         return memcmp(a, b, sizeof(*a));
1038 }
1039
1040 static inline int hash_encap_info(struct ip_tunnel_key *key)
1041 {
1042         return jhash(key, sizeof(*key), 0);
1043 }
1044
1045 static int mlx5e_route_lookup_ipv4(struct mlx5e_priv *priv,
1046                                    struct net_device *mirred_dev,
1047                                    struct net_device **out_dev,
1048                                    struct flowi4 *fl4,
1049                                    struct neighbour **out_n,
1050                                    int *out_ttl)
1051 {
1052         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1053         struct rtable *rt;
1054         struct neighbour *n = NULL;
1055
1056 #if IS_ENABLED(CONFIG_INET)
1057         int ret;
1058
1059         rt = ip_route_output_key(dev_net(mirred_dev), fl4);
1060         ret = PTR_ERR_OR_ZERO(rt);
1061         if (ret)
1062                 return ret;
1063 #else
1064         return -EOPNOTSUPP;
1065 #endif
1066         /* if the egress device isn't on the same HW e-switch, we use the uplink */
1067         if (!switchdev_port_same_parent_id(priv->netdev, rt->dst.dev))
1068                 *out_dev = mlx5_eswitch_get_uplink_netdev(esw);
1069         else
1070                 *out_dev = rt->dst.dev;
1071
1072         *out_ttl = ip4_dst_hoplimit(&rt->dst);
1073         n = dst_neigh_lookup(&rt->dst, &fl4->daddr);
1074         ip_rt_put(rt);
1075         if (!n)
1076                 return -ENOMEM;
1077
1078         *out_n = n;
1079         return 0;
1080 }
1081
1082 static int mlx5e_route_lookup_ipv6(struct mlx5e_priv *priv,
1083                                    struct net_device *mirred_dev,
1084                                    struct net_device **out_dev,
1085                                    struct flowi6 *fl6,
1086                                    struct neighbour **out_n,
1087                                    int *out_ttl)
1088 {
1089         struct neighbour *n = NULL;
1090         struct dst_entry *dst;
1091
1092 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
1093         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1094         int ret;
1095
1096         dst = ip6_route_output(dev_net(mirred_dev), NULL, fl6);
1097         ret = dst->error;
1098         if (ret) {
1099                 dst_release(dst);
1100                 return ret;
1101         }
1102
1103         *out_ttl = ip6_dst_hoplimit(dst);
1104
1105         /* if the egress device isn't on the same HW e-switch, we use the uplink */
1106         if (!switchdev_port_same_parent_id(priv->netdev, dst->dev))
1107                 *out_dev = mlx5_eswitch_get_uplink_netdev(esw);
1108         else
1109                 *out_dev = dst->dev;
1110 #else
1111         return -EOPNOTSUPP;
1112 #endif
1113
1114         n = dst_neigh_lookup(dst, &fl6->daddr);
1115         dst_release(dst);
1116         if (!n)
1117                 return -ENOMEM;
1118
1119         *out_n = n;
1120         return 0;
1121 }
1122
1123 static int gen_vxlan_header_ipv4(struct net_device *out_dev,
1124                                  char buf[],
1125                                  unsigned char h_dest[ETH_ALEN],
1126                                  int ttl,
1127                                  __be32 daddr,
1128                                  __be32 saddr,
1129                                  __be16 udp_dst_port,
1130                                  __be32 vx_vni)
1131 {
1132         int encap_size = VXLAN_HLEN + sizeof(struct iphdr) + ETH_HLEN;
1133         struct ethhdr *eth = (struct ethhdr *)buf;
1134         struct iphdr  *ip = (struct iphdr *)((char *)eth + sizeof(struct ethhdr));
1135         struct udphdr *udp = (struct udphdr *)((char *)ip + sizeof(struct iphdr));
1136         struct vxlanhdr *vxh = (struct vxlanhdr *)((char *)udp + sizeof(struct udphdr));
1137
1138         memset(buf, 0, encap_size);
1139
1140         ether_addr_copy(eth->h_dest, h_dest);
1141         ether_addr_copy(eth->h_source, out_dev->dev_addr);
1142         eth->h_proto = htons(ETH_P_IP);
1143
1144         ip->daddr = daddr;
1145         ip->saddr = saddr;
1146
1147         ip->ttl = ttl;
1148         ip->protocol = IPPROTO_UDP;
1149         ip->version = 0x4;
1150         ip->ihl = 0x5;
1151
1152         udp->dest = udp_dst_port;
1153         vxh->vx_flags = VXLAN_HF_VNI;
1154         vxh->vx_vni = vxlan_vni_field(vx_vni);
1155
1156         return encap_size;
1157 }
1158
1159 static int gen_vxlan_header_ipv6(struct net_device *out_dev,
1160                                  char buf[],
1161                                  unsigned char h_dest[ETH_ALEN],
1162                                  int ttl,
1163                                  struct in6_addr *daddr,
1164                                  struct in6_addr *saddr,
1165                                  __be16 udp_dst_port,
1166                                  __be32 vx_vni)
1167 {
1168         int encap_size = VXLAN_HLEN + sizeof(struct ipv6hdr) + ETH_HLEN;
1169         struct ethhdr *eth = (struct ethhdr *)buf;
1170         struct ipv6hdr *ip6h = (struct ipv6hdr *)((char *)eth + sizeof(struct ethhdr));
1171         struct udphdr *udp = (struct udphdr *)((char *)ip6h + sizeof(struct ipv6hdr));
1172         struct vxlanhdr *vxh = (struct vxlanhdr *)((char *)udp + sizeof(struct udphdr));
1173
1174         memset(buf, 0, encap_size);
1175
1176         ether_addr_copy(eth->h_dest, h_dest);
1177         ether_addr_copy(eth->h_source, out_dev->dev_addr);
1178         eth->h_proto = htons(ETH_P_IPV6);
1179
1180         ip6_flow_hdr(ip6h, 0, 0);
1181         /* the HW fills up ipv6 payload len */
1182         ip6h->nexthdr     = IPPROTO_UDP;
1183         ip6h->hop_limit   = ttl;
1184         ip6h->daddr       = *daddr;
1185         ip6h->saddr       = *saddr;
1186
1187         udp->dest = udp_dst_port;
1188         vxh->vx_flags = VXLAN_HF_VNI;
1189         vxh->vx_vni = vxlan_vni_field(vx_vni);
1190
1191         return encap_size;
1192 }
1193
1194 static int mlx5e_create_encap_header_ipv4(struct mlx5e_priv *priv,
1195                                           struct net_device *mirred_dev,
1196                                           struct mlx5_encap_entry *e,
1197                                           struct net_device **out_dev)
1198 {
1199         int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
1200         struct ip_tunnel_key *tun_key = &e->tun_info.key;
1201         int encap_size, ttl, err;
1202         struct neighbour *n = NULL;
1203         struct flowi4 fl4 = {};
1204         char *encap_header;
1205
1206         encap_header = kzalloc(max_encap_size, GFP_KERNEL);
1207         if (!encap_header)
1208                 return -ENOMEM;
1209
1210         switch (e->tunnel_type) {
1211         case MLX5_HEADER_TYPE_VXLAN:
1212                 fl4.flowi4_proto = IPPROTO_UDP;
1213                 fl4.fl4_dport = tun_key->tp_dst;
1214                 break;
1215         default:
1216                 err = -EOPNOTSUPP;
1217                 goto out;
1218         }
1219         fl4.flowi4_tos = tun_key->tos;
1220         fl4.daddr = tun_key->u.ipv4.dst;
1221         fl4.saddr = tun_key->u.ipv4.src;
1222
1223         err = mlx5e_route_lookup_ipv4(priv, mirred_dev, out_dev,
1224                                       &fl4, &n, &ttl);
1225         if (err)
1226                 goto out;
1227
1228         if (!(n->nud_state & NUD_VALID)) {
1229                 pr_warn("%s: can't offload, neighbour to %pI4 invalid\n", __func__, &fl4.daddr);
1230                 err = -EOPNOTSUPP;
1231                 goto out;
1232         }
1233
1234         e->n = n;
1235         e->out_dev = *out_dev;
1236
1237         neigh_ha_snapshot(e->h_dest, n, *out_dev);
1238
1239         switch (e->tunnel_type) {
1240         case MLX5_HEADER_TYPE_VXLAN:
1241                 encap_size = gen_vxlan_header_ipv4(*out_dev, encap_header,
1242                                                    e->h_dest, ttl,
1243                                                    fl4.daddr,
1244                                                    fl4.saddr, tun_key->tp_dst,
1245                                                    tunnel_id_to_key32(tun_key->tun_id));
1246                 break;
1247         default:
1248                 err = -EOPNOTSUPP;
1249                 goto out;
1250         }
1251
1252         err = mlx5_encap_alloc(priv->mdev, e->tunnel_type,
1253                                encap_size, encap_header, &e->encap_id);
1254 out:
1255         if (err && n)
1256                 neigh_release(n);
1257         kfree(encap_header);
1258         return err;
1259 }
1260
1261 static int mlx5e_create_encap_header_ipv6(struct mlx5e_priv *priv,
1262                                           struct net_device *mirred_dev,
1263                                           struct mlx5_encap_entry *e,
1264                                           struct net_device **out_dev)
1265
1266 {
1267         int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
1268         struct ip_tunnel_key *tun_key = &e->tun_info.key;
1269         int encap_size, err, ttl = 0;
1270         struct neighbour *n = NULL;
1271         struct flowi6 fl6 = {};
1272         char *encap_header;
1273
1274         encap_header = kzalloc(max_encap_size, GFP_KERNEL);
1275         if (!encap_header)
1276                 return -ENOMEM;
1277
1278         switch (e->tunnel_type) {
1279         case MLX5_HEADER_TYPE_VXLAN:
1280                 fl6.flowi6_proto = IPPROTO_UDP;
1281                 fl6.fl6_dport = tun_key->tp_dst;
1282                 break;
1283         default:
1284                 err = -EOPNOTSUPP;
1285                 goto out;
1286         }
1287
1288         fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tun_key->tos), tun_key->label);
1289         fl6.daddr = tun_key->u.ipv6.dst;
1290         fl6.saddr = tun_key->u.ipv6.src;
1291
1292         err = mlx5e_route_lookup_ipv6(priv, mirred_dev, out_dev,
1293                                       &fl6, &n, &ttl);
1294         if (err)
1295                 goto out;
1296
1297         if (!(n->nud_state & NUD_VALID)) {
1298                 pr_warn("%s: can't offload, neighbour to %pI6 invalid\n", __func__, &fl6.daddr);
1299                 err = -EOPNOTSUPP;
1300                 goto out;
1301         }
1302
1303         e->n = n;
1304         e->out_dev = *out_dev;
1305
1306         neigh_ha_snapshot(e->h_dest, n, *out_dev);
1307
1308         switch (e->tunnel_type) {
1309         case MLX5_HEADER_TYPE_VXLAN:
1310                 encap_size = gen_vxlan_header_ipv6(*out_dev, encap_header,
1311                                                    e->h_dest, ttl,
1312                                                    &fl6.daddr,
1313                                                    &fl6.saddr, tun_key->tp_dst,
1314                                                    tunnel_id_to_key32(tun_key->tun_id));
1315                 break;
1316         default:
1317                 err = -EOPNOTSUPP;
1318                 goto out;
1319         }
1320
1321         err = mlx5_encap_alloc(priv->mdev, e->tunnel_type,
1322                                encap_size, encap_header, &e->encap_id);
1323 out:
1324         if (err && n)
1325                 neigh_release(n);
1326         kfree(encap_header);
1327         return err;
1328 }
1329
1330 static int mlx5e_attach_encap(struct mlx5e_priv *priv,
1331                               struct ip_tunnel_info *tun_info,
1332                               struct net_device *mirred_dev,
1333                               struct mlx5_esw_flow_attr *attr)
1334 {
1335         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1336         struct net_device *up_dev = mlx5_eswitch_get_uplink_netdev(esw);
1337         struct mlx5e_priv *up_priv = netdev_priv(up_dev);
1338         unsigned short family = ip_tunnel_info_af(tun_info);
1339         struct ip_tunnel_key *key = &tun_info->key;
1340         struct mlx5_encap_entry *e;
1341         struct net_device *out_dev;
1342         int tunnel_type, err = -EOPNOTSUPP;
1343         uintptr_t hash_key;
1344         bool found = false;
1345
1346         /* udp dst port must be set */
1347         if (!memchr_inv(&key->tp_dst, 0, sizeof(key->tp_dst)))
1348                 goto vxlan_encap_offload_err;
1349
1350         /* setting udp src port isn't supported */
1351         if (memchr_inv(&key->tp_src, 0, sizeof(key->tp_src))) {
1352 vxlan_encap_offload_err:
1353                 netdev_warn(priv->netdev,
1354                             "must set udp dst port and not set udp src port\n");
1355                 return -EOPNOTSUPP;
1356         }
1357
1358         if (mlx5e_vxlan_lookup_port(up_priv, be16_to_cpu(key->tp_dst)) &&
1359             MLX5_CAP_ESW(priv->mdev, vxlan_encap_decap)) {
1360                 tunnel_type = MLX5_HEADER_TYPE_VXLAN;
1361         } else {
1362                 netdev_warn(priv->netdev,
1363                             "%d isn't an offloaded vxlan udp dport\n", be16_to_cpu(key->tp_dst));
1364                 return -EOPNOTSUPP;
1365         }
1366
1367         hash_key = hash_encap_info(key);
1368
1369         hash_for_each_possible_rcu(esw->offloads.encap_tbl, e,
1370                                    encap_hlist, hash_key) {
1371                 if (!cmp_encap_info(&e->tun_info.key, key)) {
1372                         found = true;
1373                         break;
1374                 }
1375         }
1376
1377         if (found) {
1378                 attr->encap = e;
1379                 return 0;
1380         }
1381
1382         e = kzalloc(sizeof(*e), GFP_KERNEL);
1383         if (!e)
1384                 return -ENOMEM;
1385
1386         e->tun_info = *tun_info;
1387         e->tunnel_type = tunnel_type;
1388         INIT_LIST_HEAD(&e->flows);
1389
1390         if (family == AF_INET)
1391                 err = mlx5e_create_encap_header_ipv4(priv, mirred_dev, e, &out_dev);
1392         else if (family == AF_INET6)
1393                 err = mlx5e_create_encap_header_ipv6(priv, mirred_dev, e, &out_dev);
1394
1395         if (err)
1396                 goto out_err;
1397
1398         attr->encap = e;
1399         hash_add_rcu(esw->offloads.encap_tbl, &e->encap_hlist, hash_key);
1400
1401         return err;
1402
1403 out_err:
1404         kfree(e);
1405         return err;
1406 }
1407
1408 static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
1409                                 struct mlx5e_tc_flow *flow)
1410 {
1411         struct mlx5_esw_flow_attr *attr = flow->esw_attr;
1412         struct ip_tunnel_info *info = NULL;
1413         const struct tc_action *a;
1414         LIST_HEAD(actions);
1415         bool encap = false;
1416         int err;
1417
1418         if (tc_no_actions(exts))
1419                 return -EINVAL;
1420
1421         memset(attr, 0, sizeof(*attr));
1422         attr->in_rep = priv->ppriv;
1423
1424         tcf_exts_to_list(exts, &actions);
1425         list_for_each_entry(a, &actions, list) {
1426                 if (is_tcf_gact_shot(a)) {
1427                         attr->action |= MLX5_FLOW_CONTEXT_ACTION_DROP |
1428                                         MLX5_FLOW_CONTEXT_ACTION_COUNT;
1429                         continue;
1430                 }
1431
1432                 if (is_tcf_mirred_egress_redirect(a)) {
1433                         int ifindex = tcf_mirred_ifindex(a);
1434                         struct net_device *out_dev;
1435                         struct mlx5e_priv *out_priv;
1436
1437                         out_dev = __dev_get_by_index(dev_net(priv->netdev), ifindex);
1438
1439                         if (switchdev_port_same_parent_id(priv->netdev,
1440                                                           out_dev)) {
1441                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1442                                         MLX5_FLOW_CONTEXT_ACTION_COUNT;
1443                                 out_priv = netdev_priv(out_dev);
1444                                 attr->out_rep = out_priv->ppriv;
1445                         } else if (encap) {
1446                                 err = mlx5e_attach_encap(priv, info,
1447                                                          out_dev, attr);
1448                                 if (err)
1449                                         return err;
1450                                 list_add(&flow->encap, &attr->encap->flows);
1451                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_ENCAP |
1452                                         MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1453                                         MLX5_FLOW_CONTEXT_ACTION_COUNT;
1454                                 out_priv = netdev_priv(attr->encap->out_dev);
1455                                 attr->out_rep = out_priv->ppriv;
1456                         } else {
1457                                 pr_err("devices %s %s not on same switch HW, can't offload forwarding\n",
1458                                        priv->netdev->name, out_dev->name);
1459                                 return -EINVAL;
1460                         }
1461                         continue;
1462                 }
1463
1464                 if (is_tcf_tunnel_set(a)) {
1465                         info = tcf_tunnel_info(a);
1466                         if (info)
1467                                 encap = true;
1468                         else
1469                                 return -EOPNOTSUPP;
1470                         continue;
1471                 }
1472
1473                 if (is_tcf_vlan(a)) {
1474                         if (tcf_vlan_action(a) == TCA_VLAN_ACT_POP) {
1475                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP;
1476                         } else if (tcf_vlan_action(a) == TCA_VLAN_ACT_PUSH) {
1477                                 if (tcf_vlan_push_proto(a) != htons(ETH_P_8021Q))
1478                                         return -EOPNOTSUPP;
1479
1480                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
1481                                 attr->vlan = tcf_vlan_push_vid(a);
1482                         } else { /* action is TCA_VLAN_ACT_MODIFY */
1483                                 return -EOPNOTSUPP;
1484                         }
1485                         continue;
1486                 }
1487
1488                 if (is_tcf_tunnel_release(a)) {
1489                         attr->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
1490                         continue;
1491                 }
1492
1493                 return -EINVAL;
1494         }
1495         return 0;
1496 }
1497
1498 int mlx5e_configure_flower(struct mlx5e_priv *priv, __be16 protocol,
1499                            struct tc_cls_flower_offload *f)
1500 {
1501         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1502         struct mlx5e_tc_flow_parse_attr *parse_attr;
1503         struct mlx5e_tc_table *tc = &priv->fs.tc;
1504         struct mlx5e_tc_flow *flow;
1505         int attr_size, err = 0;
1506         u8 flow_flags = 0;
1507
1508         if (esw && esw->mode == SRIOV_OFFLOADS) {
1509                 flow_flags = MLX5E_TC_FLOW_ESWITCH;
1510                 attr_size  = sizeof(struct mlx5_esw_flow_attr);
1511         } else {
1512                 flow_flags = MLX5E_TC_FLOW_NIC;
1513                 attr_size  = sizeof(struct mlx5_nic_flow_attr);
1514         }
1515
1516         flow = kzalloc(sizeof(*flow) + attr_size, GFP_KERNEL);
1517         parse_attr = mlx5_vzalloc(sizeof(*parse_attr));
1518         if (!parse_attr || !flow) {
1519                 err = -ENOMEM;
1520                 goto err_free;
1521         }
1522
1523         flow->cookie = f->cookie;
1524         flow->flags = flow_flags;
1525
1526         err = parse_cls_flower(priv, flow, &parse_attr->spec, f);
1527         if (err < 0)
1528                 goto err_free;
1529
1530         if (flow->flags & MLX5E_TC_FLOW_ESWITCH) {
1531                 err = parse_tc_fdb_actions(priv, f->exts, flow);
1532                 if (err < 0)
1533                         goto err_free;
1534                 flow->rule = mlx5e_tc_add_fdb_flow(priv, parse_attr, flow);
1535         } else {
1536                 err = parse_tc_nic_actions(priv, f->exts, parse_attr, flow);
1537                 if (err < 0)
1538                         goto err_free;
1539                 flow->rule = mlx5e_tc_add_nic_flow(priv, parse_attr, flow);
1540         }
1541
1542         if (IS_ERR(flow->rule)) {
1543                 err = PTR_ERR(flow->rule);
1544                 goto err_free;
1545         }
1546
1547         err = rhashtable_insert_fast(&tc->ht, &flow->node,
1548                                      tc->ht_params);
1549         if (err)
1550                 goto err_del_rule;
1551
1552         goto out;
1553
1554 err_del_rule:
1555         mlx5e_tc_del_flow(priv, flow);
1556
1557 err_free:
1558         kfree(flow);
1559 out:
1560         kvfree(parse_attr);
1561         return err;
1562 }
1563
1564 int mlx5e_delete_flower(struct mlx5e_priv *priv,
1565                         struct tc_cls_flower_offload *f)
1566 {
1567         struct mlx5e_tc_flow *flow;
1568         struct mlx5e_tc_table *tc = &priv->fs.tc;
1569
1570         flow = rhashtable_lookup_fast(&tc->ht, &f->cookie,
1571                                       tc->ht_params);
1572         if (!flow)
1573                 return -EINVAL;
1574
1575         rhashtable_remove_fast(&tc->ht, &flow->node, tc->ht_params);
1576
1577         mlx5e_tc_del_flow(priv, flow);
1578
1579
1580         kfree(flow);
1581
1582         return 0;
1583 }
1584
1585 int mlx5e_stats_flower(struct mlx5e_priv *priv,
1586                        struct tc_cls_flower_offload *f)
1587 {
1588         struct mlx5e_tc_table *tc = &priv->fs.tc;
1589         struct mlx5e_tc_flow *flow;
1590         struct tc_action *a;
1591         struct mlx5_fc *counter;
1592         LIST_HEAD(actions);
1593         u64 bytes;
1594         u64 packets;
1595         u64 lastuse;
1596
1597         flow = rhashtable_lookup_fast(&tc->ht, &f->cookie,
1598                                       tc->ht_params);
1599         if (!flow)
1600                 return -EINVAL;
1601
1602         counter = mlx5_flow_rule_counter(flow->rule);
1603         if (!counter)
1604                 return 0;
1605
1606         mlx5_fc_query_cached(counter, &bytes, &packets, &lastuse);
1607
1608         preempt_disable();
1609
1610         tcf_exts_to_list(f->exts, &actions);
1611         list_for_each_entry(a, &actions, list)
1612                 tcf_action_stats_update(a, bytes, packets, lastuse);
1613
1614         preempt_enable();
1615
1616         return 0;
1617 }
1618
1619 static const struct rhashtable_params mlx5e_tc_flow_ht_params = {
1620         .head_offset = offsetof(struct mlx5e_tc_flow, node),
1621         .key_offset = offsetof(struct mlx5e_tc_flow, cookie),
1622         .key_len = sizeof(((struct mlx5e_tc_flow *)0)->cookie),
1623         .automatic_shrinking = true,
1624 };
1625
1626 int mlx5e_tc_init(struct mlx5e_priv *priv)
1627 {
1628         struct mlx5e_tc_table *tc = &priv->fs.tc;
1629
1630         tc->ht_params = mlx5e_tc_flow_ht_params;
1631         return rhashtable_init(&tc->ht, &tc->ht_params);
1632 }
1633
1634 static void _mlx5e_tc_del_flow(void *ptr, void *arg)
1635 {
1636         struct mlx5e_tc_flow *flow = ptr;
1637         struct mlx5e_priv *priv = arg;
1638
1639         mlx5e_tc_del_flow(priv, flow);
1640         kfree(flow);
1641 }
1642
1643 void mlx5e_tc_cleanup(struct mlx5e_priv *priv)
1644 {
1645         struct mlx5e_tc_table *tc = &priv->fs.tc;
1646
1647         rhashtable_free_and_destroy(&tc->ht, _mlx5e_tc_del_flow, priv);
1648
1649         if (!IS_ERR_OR_NULL(tc->t)) {
1650                 mlx5_destroy_flow_table(tc->t);
1651                 tc->t = NULL;
1652         }
1653 }