net/mlx5e: Support offloading TC NIC hairpin flows
[sfrench/cifs-2.6.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_tc.c
1 /*
2  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32
33 #include <net/flow_dissector.h>
34 #include <net/sch_generic.h>
35 #include <net/pkt_cls.h>
36 #include <net/tc_act/tc_gact.h>
37 #include <net/tc_act/tc_skbedit.h>
38 #include <linux/mlx5/fs.h>
39 #include <linux/mlx5/device.h>
40 #include <linux/rhashtable.h>
41 #include <net/switchdev.h>
42 #include <net/tc_act/tc_mirred.h>
43 #include <net/tc_act/tc_vlan.h>
44 #include <net/tc_act/tc_tunnel_key.h>
45 #include <net/tc_act/tc_pedit.h>
46 #include <net/tc_act/tc_csum.h>
47 #include <net/vxlan.h>
48 #include <net/arp.h>
49 #include "en.h"
50 #include "en_rep.h"
51 #include "en_tc.h"
52 #include "eswitch.h"
53 #include "vxlan.h"
54
55 struct mlx5_nic_flow_attr {
56         u32 action;
57         u32 flow_tag;
58         u32 mod_hdr_id;
59         u32 hairpin_tirn;
60 };
61
62 enum {
63         MLX5E_TC_FLOW_ESWITCH   = BIT(0),
64         MLX5E_TC_FLOW_NIC       = BIT(1),
65         MLX5E_TC_FLOW_OFFLOADED = BIT(2),
66         MLX5E_TC_FLOW_HAIRPIN   = BIT(3),
67 };
68
69 struct mlx5e_tc_flow {
70         struct rhash_head       node;
71         u64                     cookie;
72         u8                      flags;
73         struct mlx5_flow_handle *rule;
74         struct list_head        encap;   /* flows sharing the same encap ID */
75         struct list_head        mod_hdr; /* flows sharing the same mod hdr ID */
76         struct list_head        hairpin; /* flows sharing the same hairpin */
77         union {
78                 struct mlx5_esw_flow_attr esw_attr[0];
79                 struct mlx5_nic_flow_attr nic_attr[0];
80         };
81 };
82
83 struct mlx5e_tc_flow_parse_attr {
84         struct ip_tunnel_info tun_info;
85         struct mlx5_flow_spec spec;
86         int num_mod_hdr_actions;
87         void *mod_hdr_actions;
88         int mirred_ifindex;
89 };
90
91 enum {
92         MLX5_HEADER_TYPE_VXLAN = 0x0,
93         MLX5_HEADER_TYPE_NVGRE = 0x1,
94 };
95
96 #define MLX5E_TC_TABLE_NUM_GROUPS 4
97 #define MLX5E_TC_TABLE_MAX_GROUP_SIZE (1 << 16)
98
99 struct mlx5e_hairpin {
100         struct mlx5_hairpin *pair;
101
102         struct mlx5_core_dev *func_mdev;
103         u32 tdn;
104         u32 tirn;
105 };
106
107 struct mlx5e_hairpin_entry {
108         /* a node of a hash table which keeps all the  hairpin entries */
109         struct hlist_node hairpin_hlist;
110
111         /* flows sharing the same hairpin */
112         struct list_head flows;
113
114         int peer_ifindex;
115         struct mlx5e_hairpin *hp;
116 };
117
118 struct mod_hdr_key {
119         int num_actions;
120         void *actions;
121 };
122
123 struct mlx5e_mod_hdr_entry {
124         /* a node of a hash table which keeps all the mod_hdr entries */
125         struct hlist_node mod_hdr_hlist;
126
127         /* flows sharing the same mod_hdr entry */
128         struct list_head flows;
129
130         struct mod_hdr_key key;
131
132         u32 mod_hdr_id;
133 };
134
135 #define MLX5_MH_ACT_SZ MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto)
136
137 static inline u32 hash_mod_hdr_info(struct mod_hdr_key *key)
138 {
139         return jhash(key->actions,
140                      key->num_actions * MLX5_MH_ACT_SZ, 0);
141 }
142
143 static inline int cmp_mod_hdr_info(struct mod_hdr_key *a,
144                                    struct mod_hdr_key *b)
145 {
146         if (a->num_actions != b->num_actions)
147                 return 1;
148
149         return memcmp(a->actions, b->actions, a->num_actions * MLX5_MH_ACT_SZ);
150 }
151
152 static int mlx5e_attach_mod_hdr(struct mlx5e_priv *priv,
153                                 struct mlx5e_tc_flow *flow,
154                                 struct mlx5e_tc_flow_parse_attr *parse_attr)
155 {
156         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
157         int num_actions, actions_size, namespace, err;
158         struct mlx5e_mod_hdr_entry *mh;
159         struct mod_hdr_key key;
160         bool found = false;
161         u32 hash_key;
162
163         num_actions  = parse_attr->num_mod_hdr_actions;
164         actions_size = MLX5_MH_ACT_SZ * num_actions;
165
166         key.actions = parse_attr->mod_hdr_actions;
167         key.num_actions = num_actions;
168
169         hash_key = hash_mod_hdr_info(&key);
170
171         if (flow->flags & MLX5E_TC_FLOW_ESWITCH) {
172                 namespace = MLX5_FLOW_NAMESPACE_FDB;
173                 hash_for_each_possible(esw->offloads.mod_hdr_tbl, mh,
174                                        mod_hdr_hlist, hash_key) {
175                         if (!cmp_mod_hdr_info(&mh->key, &key)) {
176                                 found = true;
177                                 break;
178                         }
179                 }
180         } else {
181                 namespace = MLX5_FLOW_NAMESPACE_KERNEL;
182                 hash_for_each_possible(priv->fs.tc.mod_hdr_tbl, mh,
183                                        mod_hdr_hlist, hash_key) {
184                         if (!cmp_mod_hdr_info(&mh->key, &key)) {
185                                 found = true;
186                                 break;
187                         }
188                 }
189         }
190
191         if (found)
192                 goto attach_flow;
193
194         mh = kzalloc(sizeof(*mh) + actions_size, GFP_KERNEL);
195         if (!mh)
196                 return -ENOMEM;
197
198         mh->key.actions = (void *)mh + sizeof(*mh);
199         memcpy(mh->key.actions, key.actions, actions_size);
200         mh->key.num_actions = num_actions;
201         INIT_LIST_HEAD(&mh->flows);
202
203         err = mlx5_modify_header_alloc(priv->mdev, namespace,
204                                        mh->key.num_actions,
205                                        mh->key.actions,
206                                        &mh->mod_hdr_id);
207         if (err)
208                 goto out_err;
209
210         if (flow->flags & MLX5E_TC_FLOW_ESWITCH)
211                 hash_add(esw->offloads.mod_hdr_tbl, &mh->mod_hdr_hlist, hash_key);
212         else
213                 hash_add(priv->fs.tc.mod_hdr_tbl, &mh->mod_hdr_hlist, hash_key);
214
215 attach_flow:
216         list_add(&flow->mod_hdr, &mh->flows);
217         if (flow->flags & MLX5E_TC_FLOW_ESWITCH)
218                 flow->esw_attr->mod_hdr_id = mh->mod_hdr_id;
219         else
220                 flow->nic_attr->mod_hdr_id = mh->mod_hdr_id;
221
222         return 0;
223
224 out_err:
225         kfree(mh);
226         return err;
227 }
228
229 static void mlx5e_detach_mod_hdr(struct mlx5e_priv *priv,
230                                  struct mlx5e_tc_flow *flow)
231 {
232         struct list_head *next = flow->mod_hdr.next;
233
234         list_del(&flow->mod_hdr);
235
236         if (list_empty(next)) {
237                 struct mlx5e_mod_hdr_entry *mh;
238
239                 mh = list_entry(next, struct mlx5e_mod_hdr_entry, flows);
240
241                 mlx5_modify_header_dealloc(priv->mdev, mh->mod_hdr_id);
242                 hash_del(&mh->mod_hdr_hlist);
243                 kfree(mh);
244         }
245 }
246
247 static
248 struct mlx5_core_dev *mlx5e_hairpin_get_mdev(struct net *net, int ifindex)
249 {
250         struct net_device *netdev;
251         struct mlx5e_priv *priv;
252
253         netdev = __dev_get_by_index(net, ifindex);
254         priv = netdev_priv(netdev);
255         return priv->mdev;
256 }
257
258 static int mlx5e_hairpin_create_transport(struct mlx5e_hairpin *hp)
259 {
260         u32 in[MLX5_ST_SZ_DW(create_tir_in)] = {0};
261         void *tirc;
262         int err;
263
264         err = mlx5_core_alloc_transport_domain(hp->func_mdev, &hp->tdn);
265         if (err)
266                 goto alloc_tdn_err;
267
268         tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
269
270         MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
271         MLX5_SET(tirc, tirc, inline_rqn, hp->pair->rqn);
272         MLX5_SET(tirc, tirc, transport_domain, hp->tdn);
273
274         err = mlx5_core_create_tir(hp->func_mdev, in, MLX5_ST_SZ_BYTES(create_tir_in), &hp->tirn);
275         if (err)
276                 goto create_tir_err;
277
278         return 0;
279
280 create_tir_err:
281         mlx5_core_dealloc_transport_domain(hp->func_mdev, hp->tdn);
282 alloc_tdn_err:
283         return err;
284 }
285
286 static void mlx5e_hairpin_destroy_transport(struct mlx5e_hairpin *hp)
287 {
288         mlx5_core_destroy_tir(hp->func_mdev, hp->tirn);
289         mlx5_core_dealloc_transport_domain(hp->func_mdev, hp->tdn);
290 }
291
292 static struct mlx5e_hairpin *
293 mlx5e_hairpin_create(struct mlx5e_priv *priv, struct mlx5_hairpin_params *params,
294                      int peer_ifindex)
295 {
296         struct mlx5_core_dev *func_mdev, *peer_mdev;
297         struct mlx5e_hairpin *hp;
298         struct mlx5_hairpin *pair;
299         int err;
300
301         hp = kzalloc(sizeof(*hp), GFP_KERNEL);
302         if (!hp)
303                 return ERR_PTR(-ENOMEM);
304
305         func_mdev = priv->mdev;
306         peer_mdev = mlx5e_hairpin_get_mdev(dev_net(priv->netdev), peer_ifindex);
307
308         pair = mlx5_core_hairpin_create(func_mdev, peer_mdev, params);
309         if (IS_ERR(pair)) {
310                 err = PTR_ERR(pair);
311                 goto create_pair_err;
312         }
313         hp->pair = pair;
314         hp->func_mdev = func_mdev;
315
316         err = mlx5e_hairpin_create_transport(hp);
317         if (err)
318                 goto create_transport_err;
319
320         return hp;
321
322 create_transport_err:
323         mlx5_core_hairpin_destroy(hp->pair);
324 create_pair_err:
325         kfree(hp);
326         return ERR_PTR(err);
327 }
328
329 static void mlx5e_hairpin_destroy(struct mlx5e_hairpin *hp)
330 {
331         mlx5e_hairpin_destroy_transport(hp);
332         mlx5_core_hairpin_destroy(hp->pair);
333         kvfree(hp);
334 }
335
336 static struct mlx5e_hairpin_entry *mlx5e_hairpin_get(struct mlx5e_priv *priv,
337                                                      int peer_ifindex)
338 {
339         struct mlx5e_hairpin_entry *hpe;
340
341         hash_for_each_possible(priv->fs.tc.hairpin_tbl, hpe,
342                                hairpin_hlist, peer_ifindex) {
343                 if (hpe->peer_ifindex == peer_ifindex)
344                         return hpe;
345         }
346
347         return NULL;
348 }
349
350 static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
351                                   struct mlx5e_tc_flow *flow,
352                                   struct mlx5e_tc_flow_parse_attr *parse_attr)
353 {
354         int peer_ifindex = parse_attr->mirred_ifindex;
355         struct mlx5_hairpin_params params;
356         struct mlx5e_hairpin_entry *hpe;
357         struct mlx5e_hairpin *hp;
358         int err;
359
360         if (!MLX5_CAP_GEN(priv->mdev, hairpin)) {
361                 netdev_warn(priv->netdev, "hairpin is not supported\n");
362                 return -EOPNOTSUPP;
363         }
364
365         hpe = mlx5e_hairpin_get(priv, peer_ifindex);
366         if (hpe)
367                 goto attach_flow;
368
369         hpe = kzalloc(sizeof(*hpe), GFP_KERNEL);
370         if (!hpe)
371                 return -ENOMEM;
372
373         INIT_LIST_HEAD(&hpe->flows);
374         hpe->peer_ifindex = peer_ifindex;
375
376         params.log_data_size = 15;
377         params.log_data_size = min_t(u8, params.log_data_size,
378                                      MLX5_CAP_GEN(priv->mdev, log_max_hairpin_wq_data_sz));
379         params.log_data_size = max_t(u8, params.log_data_size,
380                                      MLX5_CAP_GEN(priv->mdev, log_min_hairpin_wq_data_sz));
381         params.q_counter = priv->q_counter;
382
383         hp = mlx5e_hairpin_create(priv, &params, peer_ifindex);
384         if (IS_ERR(hp)) {
385                 err = PTR_ERR(hp);
386                 goto create_hairpin_err;
387         }
388
389         netdev_dbg(priv->netdev, "add hairpin: tirn %x rqn %x peer %s sqn %x log data size %d\n",
390                    hp->tirn, hp->pair->rqn, hp->pair->peer_mdev->priv.name,
391                    hp->pair->sqn, params.log_data_size);
392
393         hpe->hp = hp;
394         hash_add(priv->fs.tc.hairpin_tbl, &hpe->hairpin_hlist, peer_ifindex);
395
396 attach_flow:
397         flow->nic_attr->hairpin_tirn = hpe->hp->tirn;
398         list_add(&flow->hairpin, &hpe->flows);
399         return 0;
400
401 create_hairpin_err:
402         kfree(hpe);
403         return err;
404 }
405
406 static void mlx5e_hairpin_flow_del(struct mlx5e_priv *priv,
407                                    struct mlx5e_tc_flow *flow)
408 {
409         struct list_head *next = flow->hairpin.next;
410
411         list_del(&flow->hairpin);
412
413         /* no more hairpin flows for us, release the hairpin pair */
414         if (list_empty(next)) {
415                 struct mlx5e_hairpin_entry *hpe;
416
417                 hpe = list_entry(next, struct mlx5e_hairpin_entry, flows);
418
419                 netdev_dbg(priv->netdev, "del hairpin: peer %s\n",
420                            hpe->hp->pair->peer_mdev->priv.name);
421
422                 mlx5e_hairpin_destroy(hpe->hp);
423                 hash_del(&hpe->hairpin_hlist);
424                 kfree(hpe);
425         }
426 }
427
428 static struct mlx5_flow_handle *
429 mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
430                       struct mlx5e_tc_flow_parse_attr *parse_attr,
431                       struct mlx5e_tc_flow *flow)
432 {
433         struct mlx5_nic_flow_attr *attr = flow->nic_attr;
434         struct mlx5_core_dev *dev = priv->mdev;
435         struct mlx5_flow_destination dest[2] = {};
436         struct mlx5_flow_act flow_act = {
437                 .action = attr->action,
438                 .flow_tag = attr->flow_tag,
439                 .encap_id = 0,
440         };
441         struct mlx5_fc *counter = NULL;
442         struct mlx5_flow_handle *rule;
443         bool table_created = false;
444         int err, dest_ix = 0;
445
446         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
447                 if (flow->flags & MLX5E_TC_FLOW_HAIRPIN) {
448                         err = mlx5e_hairpin_flow_add(priv, flow, parse_attr);
449                         if (err) {
450                                 rule = ERR_PTR(err);
451                                 goto err_add_hairpin_flow;
452                         }
453                         dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_TIR;
454                         dest[dest_ix].tir_num = attr->hairpin_tirn;
455                 } else {
456                         dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
457                         dest[dest_ix].ft = priv->fs.vlan.ft.t;
458                 }
459                 dest_ix++;
460         }
461
462         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
463                 counter = mlx5_fc_create(dev, true);
464                 if (IS_ERR(counter)) {
465                         rule = ERR_CAST(counter);
466                         goto err_fc_create;
467                 }
468                 dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
469                 dest[dest_ix].counter = counter;
470                 dest_ix++;
471         }
472
473         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) {
474                 err = mlx5e_attach_mod_hdr(priv, flow, parse_attr);
475                 flow_act.modify_id = attr->mod_hdr_id;
476                 kfree(parse_attr->mod_hdr_actions);
477                 if (err) {
478                         rule = ERR_PTR(err);
479                         goto err_create_mod_hdr_id;
480                 }
481         }
482
483         if (IS_ERR_OR_NULL(priv->fs.tc.t)) {
484                 int tc_grp_size, tc_tbl_size;
485                 u32 max_flow_counter;
486
487                 max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) |
488                                     MLX5_CAP_GEN(dev, max_flow_counter_15_0);
489
490                 tc_grp_size = min_t(int, max_flow_counter, MLX5E_TC_TABLE_MAX_GROUP_SIZE);
491
492                 tc_tbl_size = min_t(int, tc_grp_size * MLX5E_TC_TABLE_NUM_GROUPS,
493                                     BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev, log_max_ft_size)));
494
495                 priv->fs.tc.t =
496                         mlx5_create_auto_grouped_flow_table(priv->fs.ns,
497                                                             MLX5E_TC_PRIO,
498                                                             tc_tbl_size,
499                                                             MLX5E_TC_TABLE_NUM_GROUPS,
500                                                             0, 0);
501                 if (IS_ERR(priv->fs.tc.t)) {
502                         netdev_err(priv->netdev,
503                                    "Failed to create tc offload table\n");
504                         rule = ERR_CAST(priv->fs.tc.t);
505                         goto err_create_ft;
506                 }
507
508                 table_created = true;
509         }
510
511         parse_attr->spec.match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
512         rule = mlx5_add_flow_rules(priv->fs.tc.t, &parse_attr->spec,
513                                    &flow_act, dest, dest_ix);
514
515         if (IS_ERR(rule))
516                 goto err_add_rule;
517
518         return rule;
519
520 err_add_rule:
521         if (table_created) {
522                 mlx5_destroy_flow_table(priv->fs.tc.t);
523                 priv->fs.tc.t = NULL;
524         }
525 err_create_ft:
526         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
527                 mlx5e_detach_mod_hdr(priv, flow);
528 err_create_mod_hdr_id:
529         mlx5_fc_destroy(dev, counter);
530 err_fc_create:
531         if (flow->flags & MLX5E_TC_FLOW_HAIRPIN)
532                 mlx5e_hairpin_flow_del(priv, flow);
533 err_add_hairpin_flow:
534         return rule;
535 }
536
537 static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv,
538                                   struct mlx5e_tc_flow *flow)
539 {
540         struct mlx5_nic_flow_attr *attr = flow->nic_attr;
541         struct mlx5_fc *counter = NULL;
542
543         counter = mlx5_flow_rule_counter(flow->rule);
544         mlx5_del_flow_rules(flow->rule);
545         mlx5_fc_destroy(priv->mdev, counter);
546
547         if (!mlx5e_tc_num_filters(priv) && (priv->fs.tc.t)) {
548                 mlx5_destroy_flow_table(priv->fs.tc.t);
549                 priv->fs.tc.t = NULL;
550         }
551
552         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
553                 mlx5e_detach_mod_hdr(priv, flow);
554
555         if (flow->flags & MLX5E_TC_FLOW_HAIRPIN)
556                 mlx5e_hairpin_flow_del(priv, flow);
557 }
558
559 static void mlx5e_detach_encap(struct mlx5e_priv *priv,
560                                struct mlx5e_tc_flow *flow);
561
562 static int mlx5e_attach_encap(struct mlx5e_priv *priv,
563                               struct ip_tunnel_info *tun_info,
564                               struct net_device *mirred_dev,
565                               struct net_device **encap_dev,
566                               struct mlx5e_tc_flow *flow);
567
568 static struct mlx5_flow_handle *
569 mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv,
570                       struct mlx5e_tc_flow_parse_attr *parse_attr,
571                       struct mlx5e_tc_flow *flow)
572 {
573         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
574         struct mlx5_esw_flow_attr *attr = flow->esw_attr;
575         struct net_device *out_dev, *encap_dev = NULL;
576         struct mlx5_flow_handle *rule = NULL;
577         struct mlx5e_rep_priv *rpriv;
578         struct mlx5e_priv *out_priv;
579         int err;
580
581         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_ENCAP) {
582                 out_dev = __dev_get_by_index(dev_net(priv->netdev),
583                                              attr->parse_attr->mirred_ifindex);
584                 err = mlx5e_attach_encap(priv, &parse_attr->tun_info,
585                                          out_dev, &encap_dev, flow);
586                 if (err) {
587                         rule = ERR_PTR(err);
588                         if (err != -EAGAIN)
589                                 goto err_attach_encap;
590                 }
591                 out_priv = netdev_priv(encap_dev);
592                 rpriv = out_priv->ppriv;
593                 attr->out_rep = rpriv->rep;
594         }
595
596         err = mlx5_eswitch_add_vlan_action(esw, attr);
597         if (err) {
598                 rule = ERR_PTR(err);
599                 goto err_add_vlan;
600         }
601
602         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) {
603                 err = mlx5e_attach_mod_hdr(priv, flow, parse_attr);
604                 kfree(parse_attr->mod_hdr_actions);
605                 if (err) {
606                         rule = ERR_PTR(err);
607                         goto err_mod_hdr;
608                 }
609         }
610
611         /* we get here if (1) there's no error (rule being null) or when
612          * (2) there's an encap action and we're on -EAGAIN (no valid neigh)
613          */
614         if (rule != ERR_PTR(-EAGAIN)) {
615                 rule = mlx5_eswitch_add_offloaded_rule(esw, &parse_attr->spec, attr);
616                 if (IS_ERR(rule))
617                         goto err_add_rule;
618         }
619         return rule;
620
621 err_add_rule:
622         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
623                 mlx5e_detach_mod_hdr(priv, flow);
624 err_mod_hdr:
625         mlx5_eswitch_del_vlan_action(esw, attr);
626 err_add_vlan:
627         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_ENCAP)
628                 mlx5e_detach_encap(priv, flow);
629 err_attach_encap:
630         return rule;
631 }
632
633 static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv,
634                                   struct mlx5e_tc_flow *flow)
635 {
636         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
637         struct mlx5_esw_flow_attr *attr = flow->esw_attr;
638
639         if (flow->flags & MLX5E_TC_FLOW_OFFLOADED) {
640                 flow->flags &= ~MLX5E_TC_FLOW_OFFLOADED;
641                 mlx5_eswitch_del_offloaded_rule(esw, flow->rule, attr);
642         }
643
644         mlx5_eswitch_del_vlan_action(esw, attr);
645
646         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_ENCAP) {
647                 mlx5e_detach_encap(priv, flow);
648                 kvfree(attr->parse_attr);
649         }
650
651         if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
652                 mlx5e_detach_mod_hdr(priv, flow);
653 }
654
655 void mlx5e_tc_encap_flows_add(struct mlx5e_priv *priv,
656                               struct mlx5e_encap_entry *e)
657 {
658         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
659         struct mlx5_esw_flow_attr *esw_attr;
660         struct mlx5e_tc_flow *flow;
661         int err;
662
663         err = mlx5_encap_alloc(priv->mdev, e->tunnel_type,
664                                e->encap_size, e->encap_header,
665                                &e->encap_id);
666         if (err) {
667                 mlx5_core_warn(priv->mdev, "Failed to offload cached encapsulation header, %d\n",
668                                err);
669                 return;
670         }
671         e->flags |= MLX5_ENCAP_ENTRY_VALID;
672         mlx5e_rep_queue_neigh_stats_work(priv);
673
674         list_for_each_entry(flow, &e->flows, encap) {
675                 esw_attr = flow->esw_attr;
676                 esw_attr->encap_id = e->encap_id;
677                 flow->rule = mlx5_eswitch_add_offloaded_rule(esw, &esw_attr->parse_attr->spec, esw_attr);
678                 if (IS_ERR(flow->rule)) {
679                         err = PTR_ERR(flow->rule);
680                         mlx5_core_warn(priv->mdev, "Failed to update cached encapsulation flow, %d\n",
681                                        err);
682                         continue;
683                 }
684                 flow->flags |= MLX5E_TC_FLOW_OFFLOADED;
685         }
686 }
687
688 void mlx5e_tc_encap_flows_del(struct mlx5e_priv *priv,
689                               struct mlx5e_encap_entry *e)
690 {
691         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
692         struct mlx5e_tc_flow *flow;
693
694         list_for_each_entry(flow, &e->flows, encap) {
695                 if (flow->flags & MLX5E_TC_FLOW_OFFLOADED) {
696                         flow->flags &= ~MLX5E_TC_FLOW_OFFLOADED;
697                         mlx5_eswitch_del_offloaded_rule(esw, flow->rule, flow->esw_attr);
698                 }
699         }
700
701         if (e->flags & MLX5_ENCAP_ENTRY_VALID) {
702                 e->flags &= ~MLX5_ENCAP_ENTRY_VALID;
703                 mlx5_encap_dealloc(priv->mdev, e->encap_id);
704         }
705 }
706
707 void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe)
708 {
709         struct mlx5e_neigh *m_neigh = &nhe->m_neigh;
710         u64 bytes, packets, lastuse = 0;
711         struct mlx5e_tc_flow *flow;
712         struct mlx5e_encap_entry *e;
713         struct mlx5_fc *counter;
714         struct neigh_table *tbl;
715         bool neigh_used = false;
716         struct neighbour *n;
717
718         if (m_neigh->family == AF_INET)
719                 tbl = &arp_tbl;
720 #if IS_ENABLED(CONFIG_IPV6)
721         else if (m_neigh->family == AF_INET6)
722                 tbl = ipv6_stub->nd_tbl;
723 #endif
724         else
725                 return;
726
727         list_for_each_entry(e, &nhe->encap_list, encap_list) {
728                 if (!(e->flags & MLX5_ENCAP_ENTRY_VALID))
729                         continue;
730                 list_for_each_entry(flow, &e->flows, encap) {
731                         if (flow->flags & MLX5E_TC_FLOW_OFFLOADED) {
732                                 counter = mlx5_flow_rule_counter(flow->rule);
733                                 mlx5_fc_query_cached(counter, &bytes, &packets, &lastuse);
734                                 if (time_after((unsigned long)lastuse, nhe->reported_lastuse)) {
735                                         neigh_used = true;
736                                         break;
737                                 }
738                         }
739                 }
740         }
741
742         if (neigh_used) {
743                 nhe->reported_lastuse = jiffies;
744
745                 /* find the relevant neigh according to the cached device and
746                  * dst ip pair
747                  */
748                 n = neigh_lookup(tbl, &m_neigh->dst_ip, m_neigh->dev);
749                 if (!n) {
750                         WARN(1, "The neighbour already freed\n");
751                         return;
752                 }
753
754                 neigh_event_send(n, NULL);
755                 neigh_release(n);
756         }
757 }
758
759 static void mlx5e_detach_encap(struct mlx5e_priv *priv,
760                                struct mlx5e_tc_flow *flow)
761 {
762         struct list_head *next = flow->encap.next;
763
764         list_del(&flow->encap);
765         if (list_empty(next)) {
766                 struct mlx5e_encap_entry *e;
767
768                 e = list_entry(next, struct mlx5e_encap_entry, flows);
769                 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
770
771                 if (e->flags & MLX5_ENCAP_ENTRY_VALID)
772                         mlx5_encap_dealloc(priv->mdev, e->encap_id);
773
774                 hash_del_rcu(&e->encap_hlist);
775                 kfree(e->encap_header);
776                 kfree(e);
777         }
778 }
779
780 static void mlx5e_tc_del_flow(struct mlx5e_priv *priv,
781                               struct mlx5e_tc_flow *flow)
782 {
783         if (flow->flags & MLX5E_TC_FLOW_ESWITCH)
784                 mlx5e_tc_del_fdb_flow(priv, flow);
785         else
786                 mlx5e_tc_del_nic_flow(priv, flow);
787 }
788
789 static void parse_vxlan_attr(struct mlx5_flow_spec *spec,
790                              struct tc_cls_flower_offload *f)
791 {
792         void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
793                                        outer_headers);
794         void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
795                                        outer_headers);
796         void *misc_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
797                                     misc_parameters);
798         void *misc_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
799                                     misc_parameters);
800
801         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol);
802         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
803
804         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
805                 struct flow_dissector_key_keyid *key =
806                         skb_flow_dissector_target(f->dissector,
807                                                   FLOW_DISSECTOR_KEY_ENC_KEYID,
808                                                   f->key);
809                 struct flow_dissector_key_keyid *mask =
810                         skb_flow_dissector_target(f->dissector,
811                                                   FLOW_DISSECTOR_KEY_ENC_KEYID,
812                                                   f->mask);
813                 MLX5_SET(fte_match_set_misc, misc_c, vxlan_vni,
814                          be32_to_cpu(mask->keyid));
815                 MLX5_SET(fte_match_set_misc, misc_v, vxlan_vni,
816                          be32_to_cpu(key->keyid));
817         }
818 }
819
820 static int parse_tunnel_attr(struct mlx5e_priv *priv,
821                              struct mlx5_flow_spec *spec,
822                              struct tc_cls_flower_offload *f)
823 {
824         void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
825                                        outer_headers);
826         void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
827                                        outer_headers);
828
829         struct flow_dissector_key_control *enc_control =
830                 skb_flow_dissector_target(f->dissector,
831                                           FLOW_DISSECTOR_KEY_ENC_CONTROL,
832                                           f->key);
833
834         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
835                 struct flow_dissector_key_ports *key =
836                         skb_flow_dissector_target(f->dissector,
837                                                   FLOW_DISSECTOR_KEY_ENC_PORTS,
838                                                   f->key);
839                 struct flow_dissector_key_ports *mask =
840                         skb_flow_dissector_target(f->dissector,
841                                                   FLOW_DISSECTOR_KEY_ENC_PORTS,
842                                                   f->mask);
843                 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
844                 struct mlx5e_rep_priv *uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
845                 struct net_device *up_dev = uplink_rpriv->netdev;
846                 struct mlx5e_priv *up_priv = netdev_priv(up_dev);
847
848                 /* Full udp dst port must be given */
849                 if (memchr_inv(&mask->dst, 0xff, sizeof(mask->dst)))
850                         goto vxlan_match_offload_err;
851
852                 if (mlx5e_vxlan_lookup_port(up_priv, be16_to_cpu(key->dst)) &&
853                     MLX5_CAP_ESW(priv->mdev, vxlan_encap_decap))
854                         parse_vxlan_attr(spec, f);
855                 else {
856                         netdev_warn(priv->netdev,
857                                     "%d isn't an offloaded vxlan udp dport\n", be16_to_cpu(key->dst));
858                         return -EOPNOTSUPP;
859                 }
860
861                 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
862                          udp_dport, ntohs(mask->dst));
863                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
864                          udp_dport, ntohs(key->dst));
865
866                 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
867                          udp_sport, ntohs(mask->src));
868                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
869                          udp_sport, ntohs(key->src));
870         } else { /* udp dst port must be given */
871 vxlan_match_offload_err:
872                 netdev_warn(priv->netdev,
873                             "IP tunnel decap offload supported only for vxlan, must set UDP dport\n");
874                 return -EOPNOTSUPP;
875         }
876
877         if (enc_control->addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
878                 struct flow_dissector_key_ipv4_addrs *key =
879                         skb_flow_dissector_target(f->dissector,
880                                                   FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
881                                                   f->key);
882                 struct flow_dissector_key_ipv4_addrs *mask =
883                         skb_flow_dissector_target(f->dissector,
884                                                   FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
885                                                   f->mask);
886                 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
887                          src_ipv4_src_ipv6.ipv4_layout.ipv4,
888                          ntohl(mask->src));
889                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
890                          src_ipv4_src_ipv6.ipv4_layout.ipv4,
891                          ntohl(key->src));
892
893                 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
894                          dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
895                          ntohl(mask->dst));
896                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
897                          dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
898                          ntohl(key->dst));
899
900                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ethertype);
901                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, ETH_P_IP);
902         } else if (enc_control->addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
903                 struct flow_dissector_key_ipv6_addrs *key =
904                         skb_flow_dissector_target(f->dissector,
905                                                   FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS,
906                                                   f->key);
907                 struct flow_dissector_key_ipv6_addrs *mask =
908                         skb_flow_dissector_target(f->dissector,
909                                                   FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS,
910                                                   f->mask);
911
912                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
913                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
914                        &mask->src, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6));
915                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
916                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
917                        &key->src, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6));
918
919                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
920                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
921                        &mask->dst, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6));
922                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
923                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
924                        &key->dst, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6));
925
926                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ethertype);
927                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, ETH_P_IPV6);
928         }
929
930         /* Enforce DMAC when offloading incoming tunneled flows.
931          * Flow counters require a match on the DMAC.
932          */
933         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, dmac_47_16);
934         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, dmac_15_0);
935         ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
936                                      dmac_47_16), priv->netdev->dev_addr);
937
938         /* let software handle IP fragments */
939         MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
940         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
941
942         return 0;
943 }
944
945 static int __parse_cls_flower(struct mlx5e_priv *priv,
946                               struct mlx5_flow_spec *spec,
947                               struct tc_cls_flower_offload *f,
948                               u8 *min_inline)
949 {
950         void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
951                                        outer_headers);
952         void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
953                                        outer_headers);
954         u16 addr_type = 0;
955         u8 ip_proto = 0;
956
957         *min_inline = MLX5_INLINE_MODE_L2;
958
959         if (f->dissector->used_keys &
960             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
961               BIT(FLOW_DISSECTOR_KEY_BASIC) |
962               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
963               BIT(FLOW_DISSECTOR_KEY_VLAN) |
964               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
965               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
966               BIT(FLOW_DISSECTOR_KEY_PORTS) |
967               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) |
968               BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
969               BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
970               BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) |
971               BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) |
972               BIT(FLOW_DISSECTOR_KEY_TCP) |
973               BIT(FLOW_DISSECTOR_KEY_IP))) {
974                 netdev_warn(priv->netdev, "Unsupported key used: 0x%x\n",
975                             f->dissector->used_keys);
976                 return -EOPNOTSUPP;
977         }
978
979         if ((dissector_uses_key(f->dissector,
980                                 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) ||
981              dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID) ||
982              dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) &&
983             dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
984                 struct flow_dissector_key_control *key =
985                         skb_flow_dissector_target(f->dissector,
986                                                   FLOW_DISSECTOR_KEY_ENC_CONTROL,
987                                                   f->key);
988                 switch (key->addr_type) {
989                 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
990                 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
991                         if (parse_tunnel_attr(priv, spec, f))
992                                 return -EOPNOTSUPP;
993                         break;
994                 default:
995                         return -EOPNOTSUPP;
996                 }
997
998                 /* In decap flow, header pointers should point to the inner
999                  * headers, outer header were already set by parse_tunnel_attr
1000                  */
1001                 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1002                                          inner_headers);
1003                 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1004                                          inner_headers);
1005         }
1006
1007         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
1008                 struct flow_dissector_key_control *key =
1009                         skb_flow_dissector_target(f->dissector,
1010                                                   FLOW_DISSECTOR_KEY_CONTROL,
1011                                                   f->key);
1012
1013                 struct flow_dissector_key_control *mask =
1014                         skb_flow_dissector_target(f->dissector,
1015                                                   FLOW_DISSECTOR_KEY_CONTROL,
1016                                                   f->mask);
1017                 addr_type = key->addr_type;
1018
1019                 if (mask->flags & FLOW_DIS_IS_FRAGMENT) {
1020                         MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
1021                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
1022                                  key->flags & FLOW_DIS_IS_FRAGMENT);
1023
1024                         /* the HW doesn't need L3 inline to match on frag=no */
1025                         if (key->flags & FLOW_DIS_IS_FRAGMENT)
1026                                 *min_inline = MLX5_INLINE_MODE_IP;
1027                 }
1028         }
1029
1030         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
1031                 struct flow_dissector_key_basic *key =
1032                         skb_flow_dissector_target(f->dissector,
1033                                                   FLOW_DISSECTOR_KEY_BASIC,
1034                                                   f->key);
1035                 struct flow_dissector_key_basic *mask =
1036                         skb_flow_dissector_target(f->dissector,
1037                                                   FLOW_DISSECTOR_KEY_BASIC,
1038                                                   f->mask);
1039                 ip_proto = key->ip_proto;
1040
1041                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype,
1042                          ntohs(mask->n_proto));
1043                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
1044                          ntohs(key->n_proto));
1045
1046                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
1047                          mask->ip_proto);
1048                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
1049                          key->ip_proto);
1050
1051                 if (mask->ip_proto)
1052                         *min_inline = MLX5_INLINE_MODE_IP;
1053         }
1054
1055         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
1056                 struct flow_dissector_key_eth_addrs *key =
1057                         skb_flow_dissector_target(f->dissector,
1058                                                   FLOW_DISSECTOR_KEY_ETH_ADDRS,
1059                                                   f->key);
1060                 struct flow_dissector_key_eth_addrs *mask =
1061                         skb_flow_dissector_target(f->dissector,
1062                                                   FLOW_DISSECTOR_KEY_ETH_ADDRS,
1063                                                   f->mask);
1064
1065                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
1066                                              dmac_47_16),
1067                                 mask->dst);
1068                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1069                                              dmac_47_16),
1070                                 key->dst);
1071
1072                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
1073                                              smac_47_16),
1074                                 mask->src);
1075                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1076                                              smac_47_16),
1077                                 key->src);
1078         }
1079
1080         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
1081                 struct flow_dissector_key_vlan *key =
1082                         skb_flow_dissector_target(f->dissector,
1083                                                   FLOW_DISSECTOR_KEY_VLAN,
1084                                                   f->key);
1085                 struct flow_dissector_key_vlan *mask =
1086                         skb_flow_dissector_target(f->dissector,
1087                                                   FLOW_DISSECTOR_KEY_VLAN,
1088                                                   f->mask);
1089                 if (mask->vlan_id || mask->vlan_priority) {
1090                         MLX5_SET(fte_match_set_lyr_2_4, headers_c, cvlan_tag, 1);
1091                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
1092
1093                         MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_vid, mask->vlan_id);
1094                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, key->vlan_id);
1095
1096                         MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_prio, mask->vlan_priority);
1097                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, key->vlan_priority);
1098                 }
1099         }
1100
1101         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
1102                 struct flow_dissector_key_ipv4_addrs *key =
1103                         skb_flow_dissector_target(f->dissector,
1104                                                   FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1105                                                   f->key);
1106                 struct flow_dissector_key_ipv4_addrs *mask =
1107                         skb_flow_dissector_target(f->dissector,
1108                                                   FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1109                                                   f->mask);
1110
1111                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
1112                                     src_ipv4_src_ipv6.ipv4_layout.ipv4),
1113                        &mask->src, sizeof(mask->src));
1114                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1115                                     src_ipv4_src_ipv6.ipv4_layout.ipv4),
1116                        &key->src, sizeof(key->src));
1117                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
1118                                     dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
1119                        &mask->dst, sizeof(mask->dst));
1120                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1121                                     dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
1122                        &key->dst, sizeof(key->dst));
1123
1124                 if (mask->src || mask->dst)
1125                         *min_inline = MLX5_INLINE_MODE_IP;
1126         }
1127
1128         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
1129                 struct flow_dissector_key_ipv6_addrs *key =
1130                         skb_flow_dissector_target(f->dissector,
1131                                                   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1132                                                   f->key);
1133                 struct flow_dissector_key_ipv6_addrs *mask =
1134                         skb_flow_dissector_target(f->dissector,
1135                                                   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1136                                                   f->mask);
1137
1138                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
1139                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
1140                        &mask->src, sizeof(mask->src));
1141                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1142                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
1143                        &key->src, sizeof(key->src));
1144
1145                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
1146                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
1147                        &mask->dst, sizeof(mask->dst));
1148                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1149                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
1150                        &key->dst, sizeof(key->dst));
1151
1152                 if (ipv6_addr_type(&mask->src) != IPV6_ADDR_ANY ||
1153                     ipv6_addr_type(&mask->dst) != IPV6_ADDR_ANY)
1154                         *min_inline = MLX5_INLINE_MODE_IP;
1155         }
1156
1157         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_IP)) {
1158                 struct flow_dissector_key_ip *key =
1159                         skb_flow_dissector_target(f->dissector,
1160                                                   FLOW_DISSECTOR_KEY_IP,
1161                                                   f->key);
1162                 struct flow_dissector_key_ip *mask =
1163                         skb_flow_dissector_target(f->dissector,
1164                                                   FLOW_DISSECTOR_KEY_IP,
1165                                                   f->mask);
1166
1167                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn, mask->tos & 0x3);
1168                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, key->tos & 0x3);
1169
1170                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_dscp, mask->tos >> 2);
1171                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, key->tos  >> 2);
1172
1173                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ttl_hoplimit, mask->ttl);
1174                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ttl_hoplimit, key->ttl);
1175
1176                 if (mask->ttl &&
1177                     !MLX5_CAP_ESW_FLOWTABLE_FDB(priv->mdev,
1178                                                 ft_field_support.outer_ipv4_ttl))
1179                         return -EOPNOTSUPP;
1180
1181                 if (mask->tos || mask->ttl)
1182                         *min_inline = MLX5_INLINE_MODE_IP;
1183         }
1184
1185         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) {
1186                 struct flow_dissector_key_ports *key =
1187                         skb_flow_dissector_target(f->dissector,
1188                                                   FLOW_DISSECTOR_KEY_PORTS,
1189                                                   f->key);
1190                 struct flow_dissector_key_ports *mask =
1191                         skb_flow_dissector_target(f->dissector,
1192                                                   FLOW_DISSECTOR_KEY_PORTS,
1193                                                   f->mask);
1194                 switch (ip_proto) {
1195                 case IPPROTO_TCP:
1196                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
1197                                  tcp_sport, ntohs(mask->src));
1198                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
1199                                  tcp_sport, ntohs(key->src));
1200
1201                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
1202                                  tcp_dport, ntohs(mask->dst));
1203                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
1204                                  tcp_dport, ntohs(key->dst));
1205                         break;
1206
1207                 case IPPROTO_UDP:
1208                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
1209                                  udp_sport, ntohs(mask->src));
1210                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
1211                                  udp_sport, ntohs(key->src));
1212
1213                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
1214                                  udp_dport, ntohs(mask->dst));
1215                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
1216                                  udp_dport, ntohs(key->dst));
1217                         break;
1218                 default:
1219                         netdev_err(priv->netdev,
1220                                    "Only UDP and TCP transport are supported\n");
1221                         return -EINVAL;
1222                 }
1223
1224                 if (mask->src || mask->dst)
1225                         *min_inline = MLX5_INLINE_MODE_TCP_UDP;
1226         }
1227
1228         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_TCP)) {
1229                 struct flow_dissector_key_tcp *key =
1230                         skb_flow_dissector_target(f->dissector,
1231                                                   FLOW_DISSECTOR_KEY_TCP,
1232                                                   f->key);
1233                 struct flow_dissector_key_tcp *mask =
1234                         skb_flow_dissector_target(f->dissector,
1235                                                   FLOW_DISSECTOR_KEY_TCP,
1236                                                   f->mask);
1237
1238                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_flags,
1239                          ntohs(mask->flags));
1240                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
1241                          ntohs(key->flags));
1242
1243                 if (mask->flags)
1244                         *min_inline = MLX5_INLINE_MODE_TCP_UDP;
1245         }
1246
1247         return 0;
1248 }
1249
1250 static int parse_cls_flower(struct mlx5e_priv *priv,
1251                             struct mlx5e_tc_flow *flow,
1252                             struct mlx5_flow_spec *spec,
1253                             struct tc_cls_flower_offload *f)
1254 {
1255         struct mlx5_core_dev *dev = priv->mdev;
1256         struct mlx5_eswitch *esw = dev->priv.eswitch;
1257         struct mlx5e_rep_priv *rpriv = priv->ppriv;
1258         struct mlx5_eswitch_rep *rep;
1259         u8 min_inline;
1260         int err;
1261
1262         err = __parse_cls_flower(priv, spec, f, &min_inline);
1263
1264         if (!err && (flow->flags & MLX5E_TC_FLOW_ESWITCH)) {
1265                 rep = rpriv->rep;
1266                 if (rep->vport != FDB_UPLINK_VPORT &&
1267                     (esw->offloads.inline_mode != MLX5_INLINE_MODE_NONE &&
1268                     esw->offloads.inline_mode < min_inline)) {
1269                         netdev_warn(priv->netdev,
1270                                     "Flow is not offloaded due to min inline setting, required %d actual %d\n",
1271                                     min_inline, esw->offloads.inline_mode);
1272                         return -EOPNOTSUPP;
1273                 }
1274         }
1275
1276         return err;
1277 }
1278
1279 struct pedit_headers {
1280         struct ethhdr  eth;
1281         struct iphdr   ip4;
1282         struct ipv6hdr ip6;
1283         struct tcphdr  tcp;
1284         struct udphdr  udp;
1285 };
1286
1287 static int pedit_header_offsets[] = {
1288         [TCA_PEDIT_KEY_EX_HDR_TYPE_ETH] = offsetof(struct pedit_headers, eth),
1289         [TCA_PEDIT_KEY_EX_HDR_TYPE_IP4] = offsetof(struct pedit_headers, ip4),
1290         [TCA_PEDIT_KEY_EX_HDR_TYPE_IP6] = offsetof(struct pedit_headers, ip6),
1291         [TCA_PEDIT_KEY_EX_HDR_TYPE_TCP] = offsetof(struct pedit_headers, tcp),
1292         [TCA_PEDIT_KEY_EX_HDR_TYPE_UDP] = offsetof(struct pedit_headers, udp),
1293 };
1294
1295 #define pedit_header(_ph, _htype) ((void *)(_ph) + pedit_header_offsets[_htype])
1296
1297 static int set_pedit_val(u8 hdr_type, u32 mask, u32 val, u32 offset,
1298                          struct pedit_headers *masks,
1299                          struct pedit_headers *vals)
1300 {
1301         u32 *curr_pmask, *curr_pval;
1302
1303         if (hdr_type >= __PEDIT_HDR_TYPE_MAX)
1304                 goto out_err;
1305
1306         curr_pmask = (u32 *)(pedit_header(masks, hdr_type) + offset);
1307         curr_pval  = (u32 *)(pedit_header(vals, hdr_type) + offset);
1308
1309         if (*curr_pmask & mask)  /* disallow acting twice on the same location */
1310                 goto out_err;
1311
1312         *curr_pmask |= mask;
1313         *curr_pval  |= (val & mask);
1314
1315         return 0;
1316
1317 out_err:
1318         return -EOPNOTSUPP;
1319 }
1320
1321 struct mlx5_fields {
1322         u8  field;
1323         u8  size;
1324         u32 offset;
1325 };
1326
1327 #define OFFLOAD(fw_field, size, field, off) \
1328                 {MLX5_ACTION_IN_FIELD_OUT_ ## fw_field, size, offsetof(struct pedit_headers, field) + (off)}
1329
1330 static struct mlx5_fields fields[] = {
1331         OFFLOAD(DMAC_47_16, 4, eth.h_dest[0], 0),
1332         OFFLOAD(DMAC_47_16, 4, eth.h_dest[0], 0),
1333         OFFLOAD(DMAC_15_0,  2, eth.h_dest[4], 0),
1334         OFFLOAD(SMAC_47_16, 4, eth.h_source[0], 0),
1335         OFFLOAD(SMAC_15_0,  2, eth.h_source[4], 0),
1336         OFFLOAD(ETHERTYPE,  2, eth.h_proto, 0),
1337
1338         OFFLOAD(IP_TTL, 1, ip4.ttl,   0),
1339         OFFLOAD(SIPV4,  4, ip4.saddr, 0),
1340         OFFLOAD(DIPV4,  4, ip4.daddr, 0),
1341
1342         OFFLOAD(SIPV6_127_96, 4, ip6.saddr.s6_addr32[0], 0),
1343         OFFLOAD(SIPV6_95_64,  4, ip6.saddr.s6_addr32[1], 0),
1344         OFFLOAD(SIPV6_63_32,  4, ip6.saddr.s6_addr32[2], 0),
1345         OFFLOAD(SIPV6_31_0,   4, ip6.saddr.s6_addr32[3], 0),
1346         OFFLOAD(DIPV6_127_96, 4, ip6.daddr.s6_addr32[0], 0),
1347         OFFLOAD(DIPV6_95_64,  4, ip6.daddr.s6_addr32[1], 0),
1348         OFFLOAD(DIPV6_63_32,  4, ip6.daddr.s6_addr32[2], 0),
1349         OFFLOAD(DIPV6_31_0,   4, ip6.daddr.s6_addr32[3], 0),
1350         OFFLOAD(IPV6_HOPLIMIT, 1, ip6.hop_limit, 0),
1351
1352         OFFLOAD(TCP_SPORT, 2, tcp.source,  0),
1353         OFFLOAD(TCP_DPORT, 2, tcp.dest,    0),
1354         OFFLOAD(TCP_FLAGS, 1, tcp.ack_seq, 5),
1355
1356         OFFLOAD(UDP_SPORT, 2, udp.source, 0),
1357         OFFLOAD(UDP_DPORT, 2, udp.dest,   0),
1358 };
1359
1360 /* On input attr->num_mod_hdr_actions tells how many HW actions can be parsed at
1361  * max from the SW pedit action. On success, it says how many HW actions were
1362  * actually parsed.
1363  */
1364 static int offload_pedit_fields(struct pedit_headers *masks,
1365                                 struct pedit_headers *vals,
1366                                 struct mlx5e_tc_flow_parse_attr *parse_attr)
1367 {
1368         struct pedit_headers *set_masks, *add_masks, *set_vals, *add_vals;
1369         int i, action_size, nactions, max_actions, first, last, next_z;
1370         void *s_masks_p, *a_masks_p, *vals_p;
1371         struct mlx5_fields *f;
1372         u8 cmd, field_bsize;
1373         u32 s_mask, a_mask;
1374         unsigned long mask;
1375         __be32 mask_be32;
1376         __be16 mask_be16;
1377         void *action;
1378
1379         set_masks = &masks[TCA_PEDIT_KEY_EX_CMD_SET];
1380         add_masks = &masks[TCA_PEDIT_KEY_EX_CMD_ADD];
1381         set_vals = &vals[TCA_PEDIT_KEY_EX_CMD_SET];
1382         add_vals = &vals[TCA_PEDIT_KEY_EX_CMD_ADD];
1383
1384         action_size = MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto);
1385         action = parse_attr->mod_hdr_actions;
1386         max_actions = parse_attr->num_mod_hdr_actions;
1387         nactions = 0;
1388
1389         for (i = 0; i < ARRAY_SIZE(fields); i++) {
1390                 f = &fields[i];
1391                 /* avoid seeing bits set from previous iterations */
1392                 s_mask = 0;
1393                 a_mask = 0;
1394
1395                 s_masks_p = (void *)set_masks + f->offset;
1396                 a_masks_p = (void *)add_masks + f->offset;
1397
1398                 memcpy(&s_mask, s_masks_p, f->size);
1399                 memcpy(&a_mask, a_masks_p, f->size);
1400
1401                 if (!s_mask && !a_mask) /* nothing to offload here */
1402                         continue;
1403
1404                 if (s_mask && a_mask) {
1405                         printk(KERN_WARNING "mlx5: can't set and add to the same HW field (%x)\n", f->field);
1406                         return -EOPNOTSUPP;
1407                 }
1408
1409                 if (nactions == max_actions) {
1410                         printk(KERN_WARNING "mlx5: parsed %d pedit actions, can't do more\n", nactions);
1411                         return -EOPNOTSUPP;
1412                 }
1413
1414                 if (s_mask) {
1415                         cmd  = MLX5_ACTION_TYPE_SET;
1416                         mask = s_mask;
1417                         vals_p = (void *)set_vals + f->offset;
1418                         /* clear to denote we consumed this field */
1419                         memset(s_masks_p, 0, f->size);
1420                 } else {
1421                         cmd  = MLX5_ACTION_TYPE_ADD;
1422                         mask = a_mask;
1423                         vals_p = (void *)add_vals + f->offset;
1424                         /* clear to denote we consumed this field */
1425                         memset(a_masks_p, 0, f->size);
1426                 }
1427
1428                 field_bsize = f->size * BITS_PER_BYTE;
1429
1430                 if (field_bsize == 32) {
1431                         mask_be32 = *(__be32 *)&mask;
1432                         mask = (__force unsigned long)cpu_to_le32(be32_to_cpu(mask_be32));
1433                 } else if (field_bsize == 16) {
1434                         mask_be16 = *(__be16 *)&mask;
1435                         mask = (__force unsigned long)cpu_to_le16(be16_to_cpu(mask_be16));
1436                 }
1437
1438                 first = find_first_bit(&mask, field_bsize);
1439                 next_z = find_next_zero_bit(&mask, field_bsize, first);
1440                 last  = find_last_bit(&mask, field_bsize);
1441                 if (first < next_z && next_z < last) {
1442                         printk(KERN_WARNING "mlx5: rewrite of few sub-fields (mask %lx) isn't offloaded\n",
1443                                mask);
1444                         return -EOPNOTSUPP;
1445                 }
1446
1447                 MLX5_SET(set_action_in, action, action_type, cmd);
1448                 MLX5_SET(set_action_in, action, field, f->field);
1449
1450                 if (cmd == MLX5_ACTION_TYPE_SET) {
1451                         MLX5_SET(set_action_in, action, offset, first);
1452                         /* length is num of bits to be written, zero means length of 32 */
1453                         MLX5_SET(set_action_in, action, length, (last - first + 1));
1454                 }
1455
1456                 if (field_bsize == 32)
1457                         MLX5_SET(set_action_in, action, data, ntohl(*(__be32 *)vals_p) >> first);
1458                 else if (field_bsize == 16)
1459                         MLX5_SET(set_action_in, action, data, ntohs(*(__be16 *)vals_p) >> first);
1460                 else if (field_bsize == 8)
1461                         MLX5_SET(set_action_in, action, data, *(u8 *)vals_p >> first);
1462
1463                 action += action_size;
1464                 nactions++;
1465         }
1466
1467         parse_attr->num_mod_hdr_actions = nactions;
1468         return 0;
1469 }
1470
1471 static int alloc_mod_hdr_actions(struct mlx5e_priv *priv,
1472                                  const struct tc_action *a, int namespace,
1473                                  struct mlx5e_tc_flow_parse_attr *parse_attr)
1474 {
1475         int nkeys, action_size, max_actions;
1476
1477         nkeys = tcf_pedit_nkeys(a);
1478         action_size = MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto);
1479
1480         if (namespace == MLX5_FLOW_NAMESPACE_FDB) /* FDB offloading */
1481                 max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(priv->mdev, max_modify_header_actions);
1482         else /* namespace is MLX5_FLOW_NAMESPACE_KERNEL - NIC offloading */
1483                 max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, max_modify_header_actions);
1484
1485         /* can get up to crazingly 16 HW actions in 32 bits pedit SW key */
1486         max_actions = min(max_actions, nkeys * 16);
1487
1488         parse_attr->mod_hdr_actions = kcalloc(max_actions, action_size, GFP_KERNEL);
1489         if (!parse_attr->mod_hdr_actions)
1490                 return -ENOMEM;
1491
1492         parse_attr->num_mod_hdr_actions = max_actions;
1493         return 0;
1494 }
1495
1496 static const struct pedit_headers zero_masks = {};
1497
1498 static int parse_tc_pedit_action(struct mlx5e_priv *priv,
1499                                  const struct tc_action *a, int namespace,
1500                                  struct mlx5e_tc_flow_parse_attr *parse_attr)
1501 {
1502         struct pedit_headers masks[__PEDIT_CMD_MAX], vals[__PEDIT_CMD_MAX], *cmd_masks;
1503         int nkeys, i, err = -EOPNOTSUPP;
1504         u32 mask, val, offset;
1505         u8 cmd, htype;
1506
1507         nkeys = tcf_pedit_nkeys(a);
1508
1509         memset(masks, 0, sizeof(struct pedit_headers) * __PEDIT_CMD_MAX);
1510         memset(vals,  0, sizeof(struct pedit_headers) * __PEDIT_CMD_MAX);
1511
1512         for (i = 0; i < nkeys; i++) {
1513                 htype = tcf_pedit_htype(a, i);
1514                 cmd = tcf_pedit_cmd(a, i);
1515                 err = -EOPNOTSUPP; /* can't be all optimistic */
1516
1517                 if (htype == TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK) {
1518                         printk(KERN_WARNING "mlx5: legacy pedit isn't offloaded\n");
1519                         goto out_err;
1520                 }
1521
1522                 if (cmd != TCA_PEDIT_KEY_EX_CMD_SET && cmd != TCA_PEDIT_KEY_EX_CMD_ADD) {
1523                         printk(KERN_WARNING "mlx5: pedit cmd %d isn't offloaded\n", cmd);
1524                         goto out_err;
1525                 }
1526
1527                 mask = tcf_pedit_mask(a, i);
1528                 val = tcf_pedit_val(a, i);
1529                 offset = tcf_pedit_offset(a, i);
1530
1531                 err = set_pedit_val(htype, ~mask, val, offset, &masks[cmd], &vals[cmd]);
1532                 if (err)
1533                         goto out_err;
1534         }
1535
1536         err = alloc_mod_hdr_actions(priv, a, namespace, parse_attr);
1537         if (err)
1538                 goto out_err;
1539
1540         err = offload_pedit_fields(masks, vals, parse_attr);
1541         if (err < 0)
1542                 goto out_dealloc_parsed_actions;
1543
1544         for (cmd = 0; cmd < __PEDIT_CMD_MAX; cmd++) {
1545                 cmd_masks = &masks[cmd];
1546                 if (memcmp(cmd_masks, &zero_masks, sizeof(zero_masks))) {
1547                         printk(KERN_WARNING "mlx5: attempt to offload an unsupported field (cmd %d)\n",
1548                                cmd);
1549                         print_hex_dump(KERN_WARNING, "mask: ", DUMP_PREFIX_ADDRESS,
1550                                        16, 1, cmd_masks, sizeof(zero_masks), true);
1551                         err = -EOPNOTSUPP;
1552                         goto out_dealloc_parsed_actions;
1553                 }
1554         }
1555
1556         return 0;
1557
1558 out_dealloc_parsed_actions:
1559         kfree(parse_attr->mod_hdr_actions);
1560 out_err:
1561         return err;
1562 }
1563
1564 static bool csum_offload_supported(struct mlx5e_priv *priv, u32 action, u32 update_flags)
1565 {
1566         u32 prot_flags = TCA_CSUM_UPDATE_FLAG_IPV4HDR | TCA_CSUM_UPDATE_FLAG_TCP |
1567                          TCA_CSUM_UPDATE_FLAG_UDP;
1568
1569         /*  The HW recalcs checksums only if re-writing headers */
1570         if (!(action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)) {
1571                 netdev_warn(priv->netdev,
1572                             "TC csum action is only offloaded with pedit\n");
1573                 return false;
1574         }
1575
1576         if (update_flags & ~prot_flags) {
1577                 netdev_warn(priv->netdev,
1578                             "can't offload TC csum action for some header/s - flags %#x\n",
1579                             update_flags);
1580                 return false;
1581         }
1582
1583         return true;
1584 }
1585
1586 static bool modify_header_match_supported(struct mlx5_flow_spec *spec,
1587                                           struct tcf_exts *exts)
1588 {
1589         const struct tc_action *a;
1590         bool modify_ip_header;
1591         LIST_HEAD(actions);
1592         u8 htype, ip_proto;
1593         void *headers_v;
1594         u16 ethertype;
1595         int nkeys, i;
1596
1597         headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers);
1598         ethertype = MLX5_GET(fte_match_set_lyr_2_4, headers_v, ethertype);
1599
1600         /* for non-IP we only re-write MACs, so we're okay */
1601         if (ethertype != ETH_P_IP && ethertype != ETH_P_IPV6)
1602                 goto out_ok;
1603
1604         modify_ip_header = false;
1605         tcf_exts_to_list(exts, &actions);
1606         list_for_each_entry(a, &actions, list) {
1607                 if (!is_tcf_pedit(a))
1608                         continue;
1609
1610                 nkeys = tcf_pedit_nkeys(a);
1611                 for (i = 0; i < nkeys; i++) {
1612                         htype = tcf_pedit_htype(a, i);
1613                         if (htype == TCA_PEDIT_KEY_EX_HDR_TYPE_IP4 ||
1614                             htype == TCA_PEDIT_KEY_EX_HDR_TYPE_IP6) {
1615                                 modify_ip_header = true;
1616                                 break;
1617                         }
1618                 }
1619         }
1620
1621         ip_proto = MLX5_GET(fte_match_set_lyr_2_4, headers_v, ip_protocol);
1622         if (modify_ip_header && ip_proto != IPPROTO_TCP && ip_proto != IPPROTO_UDP) {
1623                 pr_info("can't offload re-write of ip proto %d\n", ip_proto);
1624                 return false;
1625         }
1626
1627 out_ok:
1628         return true;
1629 }
1630
1631 static bool actions_match_supported(struct mlx5e_priv *priv,
1632                                     struct tcf_exts *exts,
1633                                     struct mlx5e_tc_flow_parse_attr *parse_attr,
1634                                     struct mlx5e_tc_flow *flow)
1635 {
1636         u32 actions;
1637
1638         if (flow->flags & MLX5E_TC_FLOW_ESWITCH)
1639                 actions = flow->esw_attr->action;
1640         else
1641                 actions = flow->nic_attr->action;
1642
1643         if (actions & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
1644                 return modify_header_match_supported(&parse_attr->spec, exts);
1645
1646         return true;
1647 }
1648
1649 static bool same_hw_devs(struct mlx5e_priv *priv, struct mlx5e_priv *peer_priv)
1650 {
1651         struct mlx5_core_dev *fmdev, *pmdev;
1652         u16 func_id, peer_id;
1653
1654         fmdev = priv->mdev;
1655         pmdev = peer_priv->mdev;
1656
1657         func_id = (u16)((fmdev->pdev->bus->number << 8) | PCI_SLOT(fmdev->pdev->devfn));
1658         peer_id = (u16)((pmdev->pdev->bus->number << 8) | PCI_SLOT(pmdev->pdev->devfn));
1659
1660         return (func_id == peer_id);
1661 }
1662
1663 static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
1664                                 struct mlx5e_tc_flow_parse_attr *parse_attr,
1665                                 struct mlx5e_tc_flow *flow)
1666 {
1667         struct mlx5_nic_flow_attr *attr = flow->nic_attr;
1668         const struct tc_action *a;
1669         LIST_HEAD(actions);
1670         int err;
1671
1672         if (!tcf_exts_has_actions(exts))
1673                 return -EINVAL;
1674
1675         attr->flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
1676         attr->action = 0;
1677
1678         tcf_exts_to_list(exts, &actions);
1679         list_for_each_entry(a, &actions, list) {
1680                 if (is_tcf_gact_shot(a)) {
1681                         attr->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
1682                         if (MLX5_CAP_FLOWTABLE(priv->mdev,
1683                                                flow_table_properties_nic_receive.flow_counter))
1684                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
1685                         continue;
1686                 }
1687
1688                 if (is_tcf_pedit(a)) {
1689                         err = parse_tc_pedit_action(priv, a, MLX5_FLOW_NAMESPACE_KERNEL,
1690                                                     parse_attr);
1691                         if (err)
1692                                 return err;
1693
1694                         attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR |
1695                                         MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1696                         continue;
1697                 }
1698
1699                 if (is_tcf_csum(a)) {
1700                         if (csum_offload_supported(priv, attr->action,
1701                                                    tcf_csum_update_flags(a)))
1702                                 continue;
1703
1704                         return -EOPNOTSUPP;
1705                 }
1706
1707                 if (is_tcf_mirred_egress_redirect(a)) {
1708                         struct net_device *peer_dev = tcf_mirred_dev(a);
1709
1710                         if (priv->netdev->netdev_ops == peer_dev->netdev_ops &&
1711                             same_hw_devs(priv, netdev_priv(peer_dev))) {
1712                                 parse_attr->mirred_ifindex = peer_dev->ifindex;
1713                                 flow->flags |= MLX5E_TC_FLOW_HAIRPIN;
1714                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1715                                                 MLX5_FLOW_CONTEXT_ACTION_COUNT;
1716                         } else {
1717                                 netdev_warn(priv->netdev, "device %s not on same HW, can't offload\n",
1718                                             peer_dev->name);
1719                                 return -EINVAL;
1720                         }
1721                         continue;
1722                 }
1723
1724                 if (is_tcf_skbedit_mark(a)) {
1725                         u32 mark = tcf_skbedit_mark(a);
1726
1727                         if (mark & ~MLX5E_TC_FLOW_ID_MASK) {
1728                                 netdev_warn(priv->netdev, "Bad flow mark - only 16 bit is supported: 0x%x\n",
1729                                             mark);
1730                                 return -EINVAL;
1731                         }
1732
1733                         attr->flow_tag = mark;
1734                         attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1735                         continue;
1736                 }
1737
1738                 return -EINVAL;
1739         }
1740
1741         if (!actions_match_supported(priv, exts, parse_attr, flow))
1742                 return -EOPNOTSUPP;
1743
1744         return 0;
1745 }
1746
1747 static inline int cmp_encap_info(struct ip_tunnel_key *a,
1748                                  struct ip_tunnel_key *b)
1749 {
1750         return memcmp(a, b, sizeof(*a));
1751 }
1752
1753 static inline int hash_encap_info(struct ip_tunnel_key *key)
1754 {
1755         return jhash(key, sizeof(*key), 0);
1756 }
1757
1758 static int mlx5e_route_lookup_ipv4(struct mlx5e_priv *priv,
1759                                    struct net_device *mirred_dev,
1760                                    struct net_device **out_dev,
1761                                    struct flowi4 *fl4,
1762                                    struct neighbour **out_n,
1763                                    int *out_ttl)
1764 {
1765         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1766         struct mlx5e_rep_priv *uplink_rpriv;
1767         struct rtable *rt;
1768         struct neighbour *n = NULL;
1769
1770 #if IS_ENABLED(CONFIG_INET)
1771         int ret;
1772
1773         rt = ip_route_output_key(dev_net(mirred_dev), fl4);
1774         ret = PTR_ERR_OR_ZERO(rt);
1775         if (ret)
1776                 return ret;
1777 #else
1778         return -EOPNOTSUPP;
1779 #endif
1780         uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
1781         /* if the egress device isn't on the same HW e-switch, we use the uplink */
1782         if (!switchdev_port_same_parent_id(priv->netdev, rt->dst.dev))
1783                 *out_dev = uplink_rpriv->netdev;
1784         else
1785                 *out_dev = rt->dst.dev;
1786
1787         *out_ttl = ip4_dst_hoplimit(&rt->dst);
1788         n = dst_neigh_lookup(&rt->dst, &fl4->daddr);
1789         ip_rt_put(rt);
1790         if (!n)
1791                 return -ENOMEM;
1792
1793         *out_n = n;
1794         return 0;
1795 }
1796
1797 static int mlx5e_route_lookup_ipv6(struct mlx5e_priv *priv,
1798                                    struct net_device *mirred_dev,
1799                                    struct net_device **out_dev,
1800                                    struct flowi6 *fl6,
1801                                    struct neighbour **out_n,
1802                                    int *out_ttl)
1803 {
1804         struct neighbour *n = NULL;
1805         struct dst_entry *dst;
1806
1807 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
1808         struct mlx5e_rep_priv *uplink_rpriv;
1809         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1810         int ret;
1811
1812         ret = ipv6_stub->ipv6_dst_lookup(dev_net(mirred_dev), NULL, &dst,
1813                                          fl6);
1814         if (ret < 0)
1815                 return ret;
1816
1817         *out_ttl = ip6_dst_hoplimit(dst);
1818
1819         uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
1820         /* if the egress device isn't on the same HW e-switch, we use the uplink */
1821         if (!switchdev_port_same_parent_id(priv->netdev, dst->dev))
1822                 *out_dev = uplink_rpriv->netdev;
1823         else
1824                 *out_dev = dst->dev;
1825 #else
1826         return -EOPNOTSUPP;
1827 #endif
1828
1829         n = dst_neigh_lookup(dst, &fl6->daddr);
1830         dst_release(dst);
1831         if (!n)
1832                 return -ENOMEM;
1833
1834         *out_n = n;
1835         return 0;
1836 }
1837
1838 static void gen_vxlan_header_ipv4(struct net_device *out_dev,
1839                                   char buf[], int encap_size,
1840                                   unsigned char h_dest[ETH_ALEN],
1841                                   int ttl,
1842                                   __be32 daddr,
1843                                   __be32 saddr,
1844                                   __be16 udp_dst_port,
1845                                   __be32 vx_vni)
1846 {
1847         struct ethhdr *eth = (struct ethhdr *)buf;
1848         struct iphdr  *ip = (struct iphdr *)((char *)eth + sizeof(struct ethhdr));
1849         struct udphdr *udp = (struct udphdr *)((char *)ip + sizeof(struct iphdr));
1850         struct vxlanhdr *vxh = (struct vxlanhdr *)((char *)udp + sizeof(struct udphdr));
1851
1852         memset(buf, 0, encap_size);
1853
1854         ether_addr_copy(eth->h_dest, h_dest);
1855         ether_addr_copy(eth->h_source, out_dev->dev_addr);
1856         eth->h_proto = htons(ETH_P_IP);
1857
1858         ip->daddr = daddr;
1859         ip->saddr = saddr;
1860
1861         ip->ttl = ttl;
1862         ip->protocol = IPPROTO_UDP;
1863         ip->version = 0x4;
1864         ip->ihl = 0x5;
1865
1866         udp->dest = udp_dst_port;
1867         vxh->vx_flags = VXLAN_HF_VNI;
1868         vxh->vx_vni = vxlan_vni_field(vx_vni);
1869 }
1870
1871 static void gen_vxlan_header_ipv6(struct net_device *out_dev,
1872                                   char buf[], int encap_size,
1873                                   unsigned char h_dest[ETH_ALEN],
1874                                   int ttl,
1875                                   struct in6_addr *daddr,
1876                                   struct in6_addr *saddr,
1877                                   __be16 udp_dst_port,
1878                                   __be32 vx_vni)
1879 {
1880         struct ethhdr *eth = (struct ethhdr *)buf;
1881         struct ipv6hdr *ip6h = (struct ipv6hdr *)((char *)eth + sizeof(struct ethhdr));
1882         struct udphdr *udp = (struct udphdr *)((char *)ip6h + sizeof(struct ipv6hdr));
1883         struct vxlanhdr *vxh = (struct vxlanhdr *)((char *)udp + sizeof(struct udphdr));
1884
1885         memset(buf, 0, encap_size);
1886
1887         ether_addr_copy(eth->h_dest, h_dest);
1888         ether_addr_copy(eth->h_source, out_dev->dev_addr);
1889         eth->h_proto = htons(ETH_P_IPV6);
1890
1891         ip6_flow_hdr(ip6h, 0, 0);
1892         /* the HW fills up ipv6 payload len */
1893         ip6h->nexthdr     = IPPROTO_UDP;
1894         ip6h->hop_limit   = ttl;
1895         ip6h->daddr       = *daddr;
1896         ip6h->saddr       = *saddr;
1897
1898         udp->dest = udp_dst_port;
1899         vxh->vx_flags = VXLAN_HF_VNI;
1900         vxh->vx_vni = vxlan_vni_field(vx_vni);
1901 }
1902
1903 static int mlx5e_create_encap_header_ipv4(struct mlx5e_priv *priv,
1904                                           struct net_device *mirred_dev,
1905                                           struct mlx5e_encap_entry *e)
1906 {
1907         int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
1908         int ipv4_encap_size = ETH_HLEN + sizeof(struct iphdr) + VXLAN_HLEN;
1909         struct ip_tunnel_key *tun_key = &e->tun_info.key;
1910         struct net_device *out_dev;
1911         struct neighbour *n = NULL;
1912         struct flowi4 fl4 = {};
1913         char *encap_header;
1914         int ttl, err;
1915         u8 nud_state;
1916
1917         if (max_encap_size < ipv4_encap_size) {
1918                 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
1919                                ipv4_encap_size, max_encap_size);
1920                 return -EOPNOTSUPP;
1921         }
1922
1923         encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL);
1924         if (!encap_header)
1925                 return -ENOMEM;
1926
1927         switch (e->tunnel_type) {
1928         case MLX5_HEADER_TYPE_VXLAN:
1929                 fl4.flowi4_proto = IPPROTO_UDP;
1930                 fl4.fl4_dport = tun_key->tp_dst;
1931                 break;
1932         default:
1933                 err = -EOPNOTSUPP;
1934                 goto free_encap;
1935         }
1936         fl4.flowi4_tos = tun_key->tos;
1937         fl4.daddr = tun_key->u.ipv4.dst;
1938         fl4.saddr = tun_key->u.ipv4.src;
1939
1940         err = mlx5e_route_lookup_ipv4(priv, mirred_dev, &out_dev,
1941                                       &fl4, &n, &ttl);
1942         if (err)
1943                 goto free_encap;
1944
1945         /* used by mlx5e_detach_encap to lookup a neigh hash table
1946          * entry in the neigh hash table when a user deletes a rule
1947          */
1948         e->m_neigh.dev = n->dev;
1949         e->m_neigh.family = n->ops->family;
1950         memcpy(&e->m_neigh.dst_ip, n->primary_key, n->tbl->key_len);
1951         e->out_dev = out_dev;
1952
1953         /* It's importent to add the neigh to the hash table before checking
1954          * the neigh validity state. So if we'll get a notification, in case the
1955          * neigh changes it's validity state, we would find the relevant neigh
1956          * in the hash.
1957          */
1958         err = mlx5e_rep_encap_entry_attach(netdev_priv(out_dev), e);
1959         if (err)
1960                 goto free_encap;
1961
1962         read_lock_bh(&n->lock);
1963         nud_state = n->nud_state;
1964         ether_addr_copy(e->h_dest, n->ha);
1965         read_unlock_bh(&n->lock);
1966
1967         switch (e->tunnel_type) {
1968         case MLX5_HEADER_TYPE_VXLAN:
1969                 gen_vxlan_header_ipv4(out_dev, encap_header,
1970                                       ipv4_encap_size, e->h_dest, ttl,
1971                                       fl4.daddr,
1972                                       fl4.saddr, tun_key->tp_dst,
1973                                       tunnel_id_to_key32(tun_key->tun_id));
1974                 break;
1975         default:
1976                 err = -EOPNOTSUPP;
1977                 goto destroy_neigh_entry;
1978         }
1979         e->encap_size = ipv4_encap_size;
1980         e->encap_header = encap_header;
1981
1982         if (!(nud_state & NUD_VALID)) {
1983                 neigh_event_send(n, NULL);
1984                 err = -EAGAIN;
1985                 goto out;
1986         }
1987
1988         err = mlx5_encap_alloc(priv->mdev, e->tunnel_type,
1989                                ipv4_encap_size, encap_header, &e->encap_id);
1990         if (err)
1991                 goto destroy_neigh_entry;
1992
1993         e->flags |= MLX5_ENCAP_ENTRY_VALID;
1994         mlx5e_rep_queue_neigh_stats_work(netdev_priv(out_dev));
1995         neigh_release(n);
1996         return err;
1997
1998 destroy_neigh_entry:
1999         mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
2000 free_encap:
2001         kfree(encap_header);
2002 out:
2003         if (n)
2004                 neigh_release(n);
2005         return err;
2006 }
2007
2008 static int mlx5e_create_encap_header_ipv6(struct mlx5e_priv *priv,
2009                                           struct net_device *mirred_dev,
2010                                           struct mlx5e_encap_entry *e)
2011 {
2012         int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
2013         int ipv6_encap_size = ETH_HLEN + sizeof(struct ipv6hdr) + VXLAN_HLEN;
2014         struct ip_tunnel_key *tun_key = &e->tun_info.key;
2015         struct net_device *out_dev;
2016         struct neighbour *n = NULL;
2017         struct flowi6 fl6 = {};
2018         char *encap_header;
2019         int err, ttl = 0;
2020         u8 nud_state;
2021
2022         if (max_encap_size < ipv6_encap_size) {
2023                 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
2024                                ipv6_encap_size, max_encap_size);
2025                 return -EOPNOTSUPP;
2026         }
2027
2028         encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL);
2029         if (!encap_header)
2030                 return -ENOMEM;
2031
2032         switch (e->tunnel_type) {
2033         case MLX5_HEADER_TYPE_VXLAN:
2034                 fl6.flowi6_proto = IPPROTO_UDP;
2035                 fl6.fl6_dport = tun_key->tp_dst;
2036                 break;
2037         default:
2038                 err = -EOPNOTSUPP;
2039                 goto free_encap;
2040         }
2041
2042         fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tun_key->tos), tun_key->label);
2043         fl6.daddr = tun_key->u.ipv6.dst;
2044         fl6.saddr = tun_key->u.ipv6.src;
2045
2046         err = mlx5e_route_lookup_ipv6(priv, mirred_dev, &out_dev,
2047                                       &fl6, &n, &ttl);
2048         if (err)
2049                 goto free_encap;
2050
2051         /* used by mlx5e_detach_encap to lookup a neigh hash table
2052          * entry in the neigh hash table when a user deletes a rule
2053          */
2054         e->m_neigh.dev = n->dev;
2055         e->m_neigh.family = n->ops->family;
2056         memcpy(&e->m_neigh.dst_ip, n->primary_key, n->tbl->key_len);
2057         e->out_dev = out_dev;
2058
2059         /* It's importent to add the neigh to the hash table before checking
2060          * the neigh validity state. So if we'll get a notification, in case the
2061          * neigh changes it's validity state, we would find the relevant neigh
2062          * in the hash.
2063          */
2064         err = mlx5e_rep_encap_entry_attach(netdev_priv(out_dev), e);
2065         if (err)
2066                 goto free_encap;
2067
2068         read_lock_bh(&n->lock);
2069         nud_state = n->nud_state;
2070         ether_addr_copy(e->h_dest, n->ha);
2071         read_unlock_bh(&n->lock);
2072
2073         switch (e->tunnel_type) {
2074         case MLX5_HEADER_TYPE_VXLAN:
2075                 gen_vxlan_header_ipv6(out_dev, encap_header,
2076                                       ipv6_encap_size, e->h_dest, ttl,
2077                                       &fl6.daddr,
2078                                       &fl6.saddr, tun_key->tp_dst,
2079                                       tunnel_id_to_key32(tun_key->tun_id));
2080                 break;
2081         default:
2082                 err = -EOPNOTSUPP;
2083                 goto destroy_neigh_entry;
2084         }
2085
2086         e->encap_size = ipv6_encap_size;
2087         e->encap_header = encap_header;
2088
2089         if (!(nud_state & NUD_VALID)) {
2090                 neigh_event_send(n, NULL);
2091                 err = -EAGAIN;
2092                 goto out;
2093         }
2094
2095         err = mlx5_encap_alloc(priv->mdev, e->tunnel_type,
2096                                ipv6_encap_size, encap_header, &e->encap_id);
2097         if (err)
2098                 goto destroy_neigh_entry;
2099
2100         e->flags |= MLX5_ENCAP_ENTRY_VALID;
2101         mlx5e_rep_queue_neigh_stats_work(netdev_priv(out_dev));
2102         neigh_release(n);
2103         return err;
2104
2105 destroy_neigh_entry:
2106         mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
2107 free_encap:
2108         kfree(encap_header);
2109 out:
2110         if (n)
2111                 neigh_release(n);
2112         return err;
2113 }
2114
2115 static int mlx5e_attach_encap(struct mlx5e_priv *priv,
2116                               struct ip_tunnel_info *tun_info,
2117                               struct net_device *mirred_dev,
2118                               struct net_device **encap_dev,
2119                               struct mlx5e_tc_flow *flow)
2120 {
2121         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
2122         struct mlx5e_rep_priv *uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw,
2123                                                                            REP_ETH);
2124         struct net_device *up_dev = uplink_rpriv->netdev;
2125         unsigned short family = ip_tunnel_info_af(tun_info);
2126         struct mlx5e_priv *up_priv = netdev_priv(up_dev);
2127         struct mlx5_esw_flow_attr *attr = flow->esw_attr;
2128         struct ip_tunnel_key *key = &tun_info->key;
2129         struct mlx5e_encap_entry *e;
2130         int tunnel_type, err = 0;
2131         uintptr_t hash_key;
2132         bool found = false;
2133
2134         /* udp dst port must be set */
2135         if (!memchr_inv(&key->tp_dst, 0, sizeof(key->tp_dst)))
2136                 goto vxlan_encap_offload_err;
2137
2138         /* setting udp src port isn't supported */
2139         if (memchr_inv(&key->tp_src, 0, sizeof(key->tp_src))) {
2140 vxlan_encap_offload_err:
2141                 netdev_warn(priv->netdev,
2142                             "must set udp dst port and not set udp src port\n");
2143                 return -EOPNOTSUPP;
2144         }
2145
2146         if (mlx5e_vxlan_lookup_port(up_priv, be16_to_cpu(key->tp_dst)) &&
2147             MLX5_CAP_ESW(priv->mdev, vxlan_encap_decap)) {
2148                 tunnel_type = MLX5_HEADER_TYPE_VXLAN;
2149         } else {
2150                 netdev_warn(priv->netdev,
2151                             "%d isn't an offloaded vxlan udp dport\n", be16_to_cpu(key->tp_dst));
2152                 return -EOPNOTSUPP;
2153         }
2154
2155         hash_key = hash_encap_info(key);
2156
2157         hash_for_each_possible_rcu(esw->offloads.encap_tbl, e,
2158                                    encap_hlist, hash_key) {
2159                 if (!cmp_encap_info(&e->tun_info.key, key)) {
2160                         found = true;
2161                         break;
2162                 }
2163         }
2164
2165         /* must verify if encap is valid or not */
2166         if (found)
2167                 goto attach_flow;
2168
2169         e = kzalloc(sizeof(*e), GFP_KERNEL);
2170         if (!e)
2171                 return -ENOMEM;
2172
2173         e->tun_info = *tun_info;
2174         e->tunnel_type = tunnel_type;
2175         INIT_LIST_HEAD(&e->flows);
2176
2177         if (family == AF_INET)
2178                 err = mlx5e_create_encap_header_ipv4(priv, mirred_dev, e);
2179         else if (family == AF_INET6)
2180                 err = mlx5e_create_encap_header_ipv6(priv, mirred_dev, e);
2181
2182         if (err && err != -EAGAIN)
2183                 goto out_err;
2184
2185         hash_add_rcu(esw->offloads.encap_tbl, &e->encap_hlist, hash_key);
2186
2187 attach_flow:
2188         list_add(&flow->encap, &e->flows);
2189         *encap_dev = e->out_dev;
2190         if (e->flags & MLX5_ENCAP_ENTRY_VALID)
2191                 attr->encap_id = e->encap_id;
2192         else
2193                 err = -EAGAIN;
2194
2195         return err;
2196
2197 out_err:
2198         kfree(e);
2199         return err;
2200 }
2201
2202 static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
2203                                 struct mlx5e_tc_flow_parse_attr *parse_attr,
2204                                 struct mlx5e_tc_flow *flow)
2205 {
2206         struct mlx5_esw_flow_attr *attr = flow->esw_attr;
2207         struct mlx5e_rep_priv *rpriv = priv->ppriv;
2208         struct ip_tunnel_info *info = NULL;
2209         const struct tc_action *a;
2210         LIST_HEAD(actions);
2211         bool encap = false;
2212         int err = 0;
2213
2214         if (!tcf_exts_has_actions(exts))
2215                 return -EINVAL;
2216
2217         memset(attr, 0, sizeof(*attr));
2218         attr->in_rep = rpriv->rep;
2219
2220         tcf_exts_to_list(exts, &actions);
2221         list_for_each_entry(a, &actions, list) {
2222                 if (is_tcf_gact_shot(a)) {
2223                         attr->action |= MLX5_FLOW_CONTEXT_ACTION_DROP |
2224                                         MLX5_FLOW_CONTEXT_ACTION_COUNT;
2225                         continue;
2226                 }
2227
2228                 if (is_tcf_pedit(a)) {
2229                         err = parse_tc_pedit_action(priv, a, MLX5_FLOW_NAMESPACE_FDB,
2230                                                     parse_attr);
2231                         if (err)
2232                                 return err;
2233
2234                         attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
2235                         continue;
2236                 }
2237
2238                 if (is_tcf_csum(a)) {
2239                         if (csum_offload_supported(priv, attr->action,
2240                                                    tcf_csum_update_flags(a)))
2241                                 continue;
2242
2243                         return -EOPNOTSUPP;
2244                 }
2245
2246                 if (is_tcf_mirred_egress_redirect(a)) {
2247                         struct net_device *out_dev;
2248                         struct mlx5e_priv *out_priv;
2249
2250                         out_dev = tcf_mirred_dev(a);
2251
2252                         if (switchdev_port_same_parent_id(priv->netdev,
2253                                                           out_dev)) {
2254                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
2255                                         MLX5_FLOW_CONTEXT_ACTION_COUNT;
2256                                 out_priv = netdev_priv(out_dev);
2257                                 rpriv = out_priv->ppriv;
2258                                 attr->out_rep = rpriv->rep;
2259                         } else if (encap) {
2260                                 parse_attr->mirred_ifindex = out_dev->ifindex;
2261                                 parse_attr->tun_info = *info;
2262                                 attr->parse_attr = parse_attr;
2263                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_ENCAP |
2264                                         MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
2265                                         MLX5_FLOW_CONTEXT_ACTION_COUNT;
2266                                 /* attr->out_rep is resolved when we handle encap */
2267                         } else {
2268                                 pr_err("devices %s %s not on same switch HW, can't offload forwarding\n",
2269                                        priv->netdev->name, out_dev->name);
2270                                 return -EINVAL;
2271                         }
2272                         continue;
2273                 }
2274
2275                 if (is_tcf_tunnel_set(a)) {
2276                         info = tcf_tunnel_info(a);
2277                         if (info)
2278                                 encap = true;
2279                         else
2280                                 return -EOPNOTSUPP;
2281                         continue;
2282                 }
2283
2284                 if (is_tcf_vlan(a)) {
2285                         if (tcf_vlan_action(a) == TCA_VLAN_ACT_POP) {
2286                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP;
2287                         } else if (tcf_vlan_action(a) == TCA_VLAN_ACT_PUSH) {
2288                                 if (tcf_vlan_push_proto(a) != htons(ETH_P_8021Q))
2289                                         return -EOPNOTSUPP;
2290
2291                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
2292                                 attr->vlan = tcf_vlan_push_vid(a);
2293                         } else { /* action is TCA_VLAN_ACT_MODIFY */
2294                                 return -EOPNOTSUPP;
2295                         }
2296                         continue;
2297                 }
2298
2299                 if (is_tcf_tunnel_release(a)) {
2300                         attr->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
2301                         continue;
2302                 }
2303
2304                 return -EINVAL;
2305         }
2306
2307         if (!actions_match_supported(priv, exts, parse_attr, flow))
2308                 return -EOPNOTSUPP;
2309
2310         return err;
2311 }
2312
2313 int mlx5e_configure_flower(struct mlx5e_priv *priv,
2314                            struct tc_cls_flower_offload *f)
2315 {
2316         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
2317         struct mlx5e_tc_flow_parse_attr *parse_attr;
2318         struct mlx5e_tc_table *tc = &priv->fs.tc;
2319         struct mlx5e_tc_flow *flow;
2320         int attr_size, err = 0;
2321         u8 flow_flags = 0;
2322
2323         if (esw && esw->mode == SRIOV_OFFLOADS) {
2324                 flow_flags = MLX5E_TC_FLOW_ESWITCH;
2325                 attr_size  = sizeof(struct mlx5_esw_flow_attr);
2326         } else {
2327                 flow_flags = MLX5E_TC_FLOW_NIC;
2328                 attr_size  = sizeof(struct mlx5_nic_flow_attr);
2329         }
2330
2331         flow = kzalloc(sizeof(*flow) + attr_size, GFP_KERNEL);
2332         parse_attr = kvzalloc(sizeof(*parse_attr), GFP_KERNEL);
2333         if (!parse_attr || !flow) {
2334                 err = -ENOMEM;
2335                 goto err_free;
2336         }
2337
2338         flow->cookie = f->cookie;
2339         flow->flags = flow_flags;
2340
2341         err = parse_cls_flower(priv, flow, &parse_attr->spec, f);
2342         if (err < 0)
2343                 goto err_free;
2344
2345         if (flow->flags & MLX5E_TC_FLOW_ESWITCH) {
2346                 err = parse_tc_fdb_actions(priv, f->exts, parse_attr, flow);
2347                 if (err < 0)
2348                         goto err_free;
2349                 flow->rule = mlx5e_tc_add_fdb_flow(priv, parse_attr, flow);
2350         } else {
2351                 err = parse_tc_nic_actions(priv, f->exts, parse_attr, flow);
2352                 if (err < 0)
2353                         goto err_free;
2354                 flow->rule = mlx5e_tc_add_nic_flow(priv, parse_attr, flow);
2355         }
2356
2357         if (IS_ERR(flow->rule)) {
2358                 err = PTR_ERR(flow->rule);
2359                 if (err != -EAGAIN)
2360                         goto err_free;
2361         }
2362
2363         if (err != -EAGAIN)
2364                 flow->flags |= MLX5E_TC_FLOW_OFFLOADED;
2365
2366         err = rhashtable_insert_fast(&tc->ht, &flow->node,
2367                                      tc->ht_params);
2368         if (err)
2369                 goto err_del_rule;
2370
2371         if (flow->flags & MLX5E_TC_FLOW_ESWITCH &&
2372             !(flow->esw_attr->action & MLX5_FLOW_CONTEXT_ACTION_ENCAP))
2373                 kvfree(parse_attr);
2374         return err;
2375
2376 err_del_rule:
2377         mlx5e_tc_del_flow(priv, flow);
2378
2379 err_free:
2380         kvfree(parse_attr);
2381         kfree(flow);
2382         return err;
2383 }
2384
2385 int mlx5e_delete_flower(struct mlx5e_priv *priv,
2386                         struct tc_cls_flower_offload *f)
2387 {
2388         struct mlx5e_tc_flow *flow;
2389         struct mlx5e_tc_table *tc = &priv->fs.tc;
2390
2391         flow = rhashtable_lookup_fast(&tc->ht, &f->cookie,
2392                                       tc->ht_params);
2393         if (!flow)
2394                 return -EINVAL;
2395
2396         rhashtable_remove_fast(&tc->ht, &flow->node, tc->ht_params);
2397
2398         mlx5e_tc_del_flow(priv, flow);
2399
2400         kfree(flow);
2401
2402         return 0;
2403 }
2404
2405 int mlx5e_stats_flower(struct mlx5e_priv *priv,
2406                        struct tc_cls_flower_offload *f)
2407 {
2408         struct mlx5e_tc_table *tc = &priv->fs.tc;
2409         struct mlx5e_tc_flow *flow;
2410         struct mlx5_fc *counter;
2411         u64 bytes;
2412         u64 packets;
2413         u64 lastuse;
2414
2415         flow = rhashtable_lookup_fast(&tc->ht, &f->cookie,
2416                                       tc->ht_params);
2417         if (!flow)
2418                 return -EINVAL;
2419
2420         if (!(flow->flags & MLX5E_TC_FLOW_OFFLOADED))
2421                 return 0;
2422
2423         counter = mlx5_flow_rule_counter(flow->rule);
2424         if (!counter)
2425                 return 0;
2426
2427         mlx5_fc_query_cached(counter, &bytes, &packets, &lastuse);
2428
2429         tcf_exts_stats_update(f->exts, bytes, packets, lastuse);
2430
2431         return 0;
2432 }
2433
2434 static const struct rhashtable_params mlx5e_tc_flow_ht_params = {
2435         .head_offset = offsetof(struct mlx5e_tc_flow, node),
2436         .key_offset = offsetof(struct mlx5e_tc_flow, cookie),
2437         .key_len = sizeof(((struct mlx5e_tc_flow *)0)->cookie),
2438         .automatic_shrinking = true,
2439 };
2440
2441 int mlx5e_tc_init(struct mlx5e_priv *priv)
2442 {
2443         struct mlx5e_tc_table *tc = &priv->fs.tc;
2444
2445         hash_init(tc->mod_hdr_tbl);
2446         hash_init(tc->hairpin_tbl);
2447
2448         tc->ht_params = mlx5e_tc_flow_ht_params;
2449         return rhashtable_init(&tc->ht, &tc->ht_params);
2450 }
2451
2452 static void _mlx5e_tc_del_flow(void *ptr, void *arg)
2453 {
2454         struct mlx5e_tc_flow *flow = ptr;
2455         struct mlx5e_priv *priv = arg;
2456
2457         mlx5e_tc_del_flow(priv, flow);
2458         kfree(flow);
2459 }
2460
2461 void mlx5e_tc_cleanup(struct mlx5e_priv *priv)
2462 {
2463         struct mlx5e_tc_table *tc = &priv->fs.tc;
2464
2465         rhashtable_free_and_destroy(&tc->ht, _mlx5e_tc_del_flow, priv);
2466
2467         if (!IS_ERR_OR_NULL(tc->t)) {
2468                 mlx5_destroy_flow_table(tc->t);
2469                 tc->t = NULL;
2470         }
2471 }