net/mlx5e: Enlarge the NIC TC offload steering prio to support two levels
[sfrench/cifs-2.6.git] / drivers / net / ethernet / mellanox / mlx5 / core / fs_core.c
1 /*
2  * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32
33 #include <linux/mutex.h>
34 #include <linux/mlx5/driver.h>
35
36 #include "mlx5_core.h"
37 #include "fs_core.h"
38 #include "fs_cmd.h"
39 #include "diag/fs_tracepoint.h"
40
41 #define INIT_TREE_NODE_ARRAY_SIZE(...)  (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\
42                                          sizeof(struct init_tree_node))
43
44 #define ADD_PRIO(num_prios_val, min_level_val, num_levels_val, caps_val,\
45                  ...) {.type = FS_TYPE_PRIO,\
46         .min_ft_level = min_level_val,\
47         .num_levels = num_levels_val,\
48         .num_leaf_prios = num_prios_val,\
49         .caps = caps_val,\
50         .children = (struct init_tree_node[]) {__VA_ARGS__},\
51         .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
52 }
53
54 #define ADD_MULTIPLE_PRIO(num_prios_val, num_levels_val, ...)\
55         ADD_PRIO(num_prios_val, 0, num_levels_val, {},\
56                  __VA_ARGS__)\
57
58 #define ADD_NS(...) {.type = FS_TYPE_NAMESPACE,\
59         .children = (struct init_tree_node[]) {__VA_ARGS__},\
60         .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
61 }
62
63 #define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\
64                                    sizeof(long))
65
66 #define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap))
67
68 #define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \
69                                .caps = (long[]) {__VA_ARGS__} }
70
71 #define FS_CHAINING_CAPS  FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), \
72                                            FS_CAP(flow_table_properties_nic_receive.modify_root), \
73                                            FS_CAP(flow_table_properties_nic_receive.identified_miss_table_mode), \
74                                            FS_CAP(flow_table_properties_nic_receive.flow_table_modify))
75
76 #define LEFTOVERS_NUM_LEVELS 1
77 #define LEFTOVERS_NUM_PRIOS 1
78
79 #define BY_PASS_PRIO_NUM_LEVELS 1
80 #define BY_PASS_MIN_LEVEL (ETHTOOL_MIN_LEVEL + MLX5_BY_PASS_NUM_PRIOS +\
81                            LEFTOVERS_NUM_PRIOS)
82
83 #define ETHTOOL_PRIO_NUM_LEVELS 1
84 #define ETHTOOL_NUM_PRIOS 11
85 #define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS)
86 /* Vlan, mac, ttc, inner ttc, aRFS */
87 #define KERNEL_NIC_PRIO_NUM_LEVELS 5
88 #define KERNEL_NIC_NUM_PRIOS 1
89 /* One more level for tc */
90 #define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1)
91
92 #define KERNEL_NIC_TC_NUM_PRIOS  1
93 #define KERNEL_NIC_TC_NUM_LEVELS 2
94
95 #define ANCHOR_NUM_LEVELS 1
96 #define ANCHOR_NUM_PRIOS 1
97 #define ANCHOR_MIN_LEVEL (BY_PASS_MIN_LEVEL + 1)
98
99 #define OFFLOADS_MAX_FT 1
100 #define OFFLOADS_NUM_PRIOS 1
101 #define OFFLOADS_MIN_LEVEL (ANCHOR_MIN_LEVEL + 1)
102
103 #define LAG_PRIO_NUM_LEVELS 1
104 #define LAG_NUM_PRIOS 1
105 #define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + 1)
106
107 struct node_caps {
108         size_t  arr_sz;
109         long    *caps;
110 };
111
112 static struct init_tree_node {
113         enum fs_node_type       type;
114         struct init_tree_node *children;
115         int ar_size;
116         struct node_caps caps;
117         int min_ft_level;
118         int num_leaf_prios;
119         int prio;
120         int num_levels;
121 } root_fs = {
122         .type = FS_TYPE_NAMESPACE,
123         .ar_size = 7,
124         .children = (struct init_tree_node[]) {
125                 ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0,
126                          FS_CHAINING_CAPS,
127                          ADD_NS(ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS,
128                                                   BY_PASS_PRIO_NUM_LEVELS))),
129                 ADD_PRIO(0, LAG_MIN_LEVEL, 0,
130                          FS_CHAINING_CAPS,
131                          ADD_NS(ADD_MULTIPLE_PRIO(LAG_NUM_PRIOS,
132                                                   LAG_PRIO_NUM_LEVELS))),
133                 ADD_PRIO(0, OFFLOADS_MIN_LEVEL, 0, {},
134                          ADD_NS(ADD_MULTIPLE_PRIO(OFFLOADS_NUM_PRIOS, OFFLOADS_MAX_FT))),
135                 ADD_PRIO(0, ETHTOOL_MIN_LEVEL, 0,
136                          FS_CHAINING_CAPS,
137                          ADD_NS(ADD_MULTIPLE_PRIO(ETHTOOL_NUM_PRIOS,
138                                                   ETHTOOL_PRIO_NUM_LEVELS))),
139                 ADD_PRIO(0, KERNEL_MIN_LEVEL, 0, {},
140                          ADD_NS(ADD_MULTIPLE_PRIO(KERNEL_NIC_TC_NUM_PRIOS, KERNEL_NIC_TC_NUM_LEVELS),
141                                 ADD_MULTIPLE_PRIO(KERNEL_NIC_NUM_PRIOS,
142                                                   KERNEL_NIC_PRIO_NUM_LEVELS))),
143                 ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0,
144                          FS_CHAINING_CAPS,
145                          ADD_NS(ADD_MULTIPLE_PRIO(LEFTOVERS_NUM_PRIOS, LEFTOVERS_NUM_LEVELS))),
146                 ADD_PRIO(0, ANCHOR_MIN_LEVEL, 0, {},
147                          ADD_NS(ADD_MULTIPLE_PRIO(ANCHOR_NUM_PRIOS, ANCHOR_NUM_LEVELS))),
148         }
149 };
150
151 enum fs_i_lock_class {
152         FS_LOCK_GRANDPARENT,
153         FS_LOCK_PARENT,
154         FS_LOCK_CHILD
155 };
156
157 static const struct rhashtable_params rhash_fte = {
158         .key_len = FIELD_SIZEOF(struct fs_fte, val),
159         .key_offset = offsetof(struct fs_fte, val),
160         .head_offset = offsetof(struct fs_fte, hash),
161         .automatic_shrinking = true,
162         .min_size = 1,
163 };
164
165 static const struct rhashtable_params rhash_fg = {
166         .key_len = FIELD_SIZEOF(struct mlx5_flow_group, mask),
167         .key_offset = offsetof(struct mlx5_flow_group, mask),
168         .head_offset = offsetof(struct mlx5_flow_group, hash),
169         .automatic_shrinking = true,
170         .min_size = 1,
171
172 };
173
174 static void del_hw_flow_table(struct fs_node *node);
175 static void del_hw_flow_group(struct fs_node *node);
176 static void del_hw_fte(struct fs_node *node);
177 static void del_sw_flow_table(struct fs_node *node);
178 static void del_sw_flow_group(struct fs_node *node);
179 static void del_sw_fte(struct fs_node *node);
180 static void del_sw_prio(struct fs_node *node);
181 static void del_sw_ns(struct fs_node *node);
182 /* Delete rule (destination) is special case that 
183  * requires to lock the FTE for all the deletion process.
184  */
185 static void del_sw_hw_rule(struct fs_node *node);
186 static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1,
187                                 struct mlx5_flow_destination *d2);
188 static struct mlx5_flow_rule *
189 find_flow_rule(struct fs_fte *fte,
190                struct mlx5_flow_destination *dest);
191
192 static void tree_init_node(struct fs_node *node,
193                            void (*del_hw_func)(struct fs_node *),
194                            void (*del_sw_func)(struct fs_node *))
195 {
196         refcount_set(&node->refcount, 1);
197         INIT_LIST_HEAD(&node->list);
198         INIT_LIST_HEAD(&node->children);
199         init_rwsem(&node->lock);
200         node->del_hw_func = del_hw_func;
201         node->del_sw_func = del_sw_func;
202         node->active = false;
203 }
204
205 static void tree_add_node(struct fs_node *node, struct fs_node *parent)
206 {
207         if (parent)
208                 refcount_inc(&parent->refcount);
209         node->parent = parent;
210
211         /* Parent is the root */
212         if (!parent)
213                 node->root = node;
214         else
215                 node->root = parent->root;
216 }
217
218 static int tree_get_node(struct fs_node *node)
219 {
220         return refcount_inc_not_zero(&node->refcount);
221 }
222
223 static void nested_down_read_ref_node(struct fs_node *node,
224                                       enum fs_i_lock_class class)
225 {
226         if (node) {
227                 down_read_nested(&node->lock, class);
228                 refcount_inc(&node->refcount);
229         }
230 }
231
232 static void nested_down_write_ref_node(struct fs_node *node,
233                                        enum fs_i_lock_class class)
234 {
235         if (node) {
236                 down_write_nested(&node->lock, class);
237                 refcount_inc(&node->refcount);
238         }
239 }
240
241 static void down_write_ref_node(struct fs_node *node)
242 {
243         if (node) {
244                 down_write(&node->lock);
245                 refcount_inc(&node->refcount);
246         }
247 }
248
249 static void up_read_ref_node(struct fs_node *node)
250 {
251         refcount_dec(&node->refcount);
252         up_read(&node->lock);
253 }
254
255 static void up_write_ref_node(struct fs_node *node)
256 {
257         refcount_dec(&node->refcount);
258         up_write(&node->lock);
259 }
260
261 static void tree_put_node(struct fs_node *node)
262 {
263         struct fs_node *parent_node = node->parent;
264
265         if (refcount_dec_and_test(&node->refcount)) {
266                 if (node->del_hw_func)
267                         node->del_hw_func(node);
268                 if (parent_node) {
269                         /* Only root namespace doesn't have parent and we just
270                          * need to free its node.
271                          */
272                         down_write_ref_node(parent_node);
273                         list_del_init(&node->list);
274                         if (node->del_sw_func)
275                                 node->del_sw_func(node);
276                         up_write_ref_node(parent_node);
277                 } else {
278                         kfree(node);
279                 }
280                 node = NULL;
281         }
282         if (!node && parent_node)
283                 tree_put_node(parent_node);
284 }
285
286 static int tree_remove_node(struct fs_node *node)
287 {
288         if (refcount_read(&node->refcount) > 1) {
289                 refcount_dec(&node->refcount);
290                 return -EEXIST;
291         }
292         tree_put_node(node);
293         return 0;
294 }
295
296 static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns,
297                                  unsigned int prio)
298 {
299         struct fs_prio *iter_prio;
300
301         fs_for_each_prio(iter_prio, ns) {
302                 if (iter_prio->prio == prio)
303                         return iter_prio;
304         }
305
306         return NULL;
307 }
308
309 static bool check_last_reserved(const u32 *match_criteria)
310 {
311         char *match_criteria_reserved =
312                 MLX5_ADDR_OF(fte_match_param, match_criteria, MLX5_FTE_MATCH_PARAM_RESERVED);
313
314         return  !match_criteria_reserved[0] &&
315                 !memcmp(match_criteria_reserved, match_criteria_reserved + 1,
316                         MLX5_FLD_SZ_BYTES(fte_match_param,
317                                           MLX5_FTE_MATCH_PARAM_RESERVED) - 1);
318 }
319
320 static bool check_valid_mask(u8 match_criteria_enable, const u32 *match_criteria)
321 {
322         if (match_criteria_enable & ~(
323                 (1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS)   |
324                 (1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) |
325                 (1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS)))
326                 return false;
327
328         if (!(match_criteria_enable &
329               1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS)) {
330                 char *fg_type_mask = MLX5_ADDR_OF(fte_match_param,
331                                                   match_criteria, outer_headers);
332
333                 if (fg_type_mask[0] ||
334                     memcmp(fg_type_mask, fg_type_mask + 1,
335                            MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4) - 1))
336                         return false;
337         }
338
339         if (!(match_criteria_enable &
340               1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS)) {
341                 char *fg_type_mask = MLX5_ADDR_OF(fte_match_param,
342                                                   match_criteria, misc_parameters);
343
344                 if (fg_type_mask[0] ||
345                     memcmp(fg_type_mask, fg_type_mask + 1,
346                            MLX5_ST_SZ_BYTES(fte_match_set_misc) - 1))
347                         return false;
348         }
349
350         if (!(match_criteria_enable &
351               1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS)) {
352                 char *fg_type_mask = MLX5_ADDR_OF(fte_match_param,
353                                                   match_criteria, inner_headers);
354
355                 if (fg_type_mask[0] ||
356                     memcmp(fg_type_mask, fg_type_mask + 1,
357                            MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4) - 1))
358                         return false;
359         }
360
361         return check_last_reserved(match_criteria);
362 }
363
364 static bool check_valid_spec(const struct mlx5_flow_spec *spec)
365 {
366         int i;
367
368         if (!check_valid_mask(spec->match_criteria_enable, spec->match_criteria)) {
369                 pr_warn("mlx5_core: Match criteria given mismatches match_criteria_enable\n");
370                 return false;
371         }
372
373         for (i = 0; i < MLX5_ST_SZ_DW_MATCH_PARAM; i++)
374                 if (spec->match_value[i] & ~spec->match_criteria[i]) {
375                         pr_warn("mlx5_core: match_value differs from match_criteria\n");
376                         return false;
377                 }
378
379         return check_last_reserved(spec->match_value);
380 }
381
382 static struct mlx5_flow_root_namespace *find_root(struct fs_node *node)
383 {
384         struct fs_node *root;
385         struct mlx5_flow_namespace *ns;
386
387         root = node->root;
388
389         if (WARN_ON(root->type != FS_TYPE_NAMESPACE)) {
390                 pr_warn("mlx5: flow steering node is not in tree or garbaged\n");
391                 return NULL;
392         }
393
394         ns = container_of(root, struct mlx5_flow_namespace, node);
395         return container_of(ns, struct mlx5_flow_root_namespace, ns);
396 }
397
398 static inline struct mlx5_flow_steering *get_steering(struct fs_node *node)
399 {
400         struct mlx5_flow_root_namespace *root = find_root(node);
401
402         if (root)
403                 return root->dev->priv.steering;
404         return NULL;
405 }
406
407 static inline struct mlx5_core_dev *get_dev(struct fs_node *node)
408 {
409         struct mlx5_flow_root_namespace *root = find_root(node);
410
411         if (root)
412                 return root->dev;
413         return NULL;
414 }
415
416 static void del_sw_ns(struct fs_node *node)
417 {
418         kfree(node);
419 }
420
421 static void del_sw_prio(struct fs_node *node)
422 {
423         kfree(node);
424 }
425
426 static void del_hw_flow_table(struct fs_node *node)
427 {
428         struct mlx5_flow_table *ft;
429         struct mlx5_core_dev *dev;
430         int err;
431
432         fs_get_obj(ft, node);
433         dev = get_dev(&ft->node);
434
435         if (node->active) {
436                 err = mlx5_cmd_destroy_flow_table(dev, ft);
437                 if (err)
438                         mlx5_core_warn(dev, "flow steering can't destroy ft\n");
439         }
440 }
441
442 static void del_sw_flow_table(struct fs_node *node)
443 {
444         struct mlx5_flow_table *ft;
445         struct fs_prio *prio;
446
447         fs_get_obj(ft, node);
448
449         rhltable_destroy(&ft->fgs_hash);
450         fs_get_obj(prio, ft->node.parent);
451         prio->num_ft--;
452         kfree(ft);
453 }
454
455 static void del_sw_hw_rule(struct fs_node *node)
456 {
457         struct mlx5_flow_rule *rule;
458         struct mlx5_flow_table *ft;
459         struct mlx5_flow_group *fg;
460         struct fs_fte *fte;
461         int modify_mask;
462         struct mlx5_core_dev *dev = get_dev(node);
463         int err;
464         bool update_fte = false;
465
466         fs_get_obj(rule, node);
467         fs_get_obj(fte, rule->node.parent);
468         fs_get_obj(fg, fte->node.parent);
469         fs_get_obj(ft, fg->node.parent);
470         trace_mlx5_fs_del_rule(rule);
471         if (rule->sw_action == MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO) {
472                 mutex_lock(&rule->dest_attr.ft->lock);
473                 list_del(&rule->next_ft);
474                 mutex_unlock(&rule->dest_attr.ft->lock);
475         }
476
477         if (rule->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_COUNTER  &&
478             --fte->dests_size) {
479                 modify_mask = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION);
480                 fte->action &= ~MLX5_FLOW_CONTEXT_ACTION_COUNT;
481                 update_fte = true;
482                 goto out;
483         }
484
485         if ((fte->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) &&
486             --fte->dests_size) {
487                 modify_mask = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST),
488                 update_fte = true;
489         }
490 out:
491         if (update_fte && fte->dests_size) {
492                 err = mlx5_cmd_update_fte(dev, ft, fg->id, modify_mask, fte);
493                 if (err)
494                         mlx5_core_warn(dev,
495                                        "%s can't del rule fg id=%d fte_index=%d\n",
496                                        __func__, fg->id, fte->index);
497         }
498         kfree(rule);
499 }
500
501 static void del_hw_fte(struct fs_node *node)
502 {
503         struct mlx5_flow_table *ft;
504         struct mlx5_flow_group *fg;
505         struct mlx5_core_dev *dev;
506         struct fs_fte *fte;
507         int err;
508
509         fs_get_obj(fte, node);
510         fs_get_obj(fg, fte->node.parent);
511         fs_get_obj(ft, fg->node.parent);
512
513         trace_mlx5_fs_del_fte(fte);
514         dev = get_dev(&ft->node);
515         if (node->active) {
516                 err = mlx5_cmd_delete_fte(dev, ft,
517                                           fte->index);
518                 if (err)
519                         mlx5_core_warn(dev,
520                                        "flow steering can't delete fte in index %d of flow group id %d\n",
521                                        fte->index, fg->id);
522         }
523 }
524
525 static void del_sw_fte(struct fs_node *node)
526 {
527         struct mlx5_flow_steering *steering = get_steering(node);
528         struct mlx5_flow_group *fg;
529         struct fs_fte *fte;
530         int err;
531
532         fs_get_obj(fte, node);
533         fs_get_obj(fg, fte->node.parent);
534
535         err = rhashtable_remove_fast(&fg->ftes_hash,
536                                      &fte->hash,
537                                      rhash_fte);
538         WARN_ON(err);
539         ida_simple_remove(&fg->fte_allocator, fte->index - fg->start_index);
540         kmem_cache_free(steering->ftes_cache, fte);
541 }
542
543 static void del_hw_flow_group(struct fs_node *node)
544 {
545         struct mlx5_flow_group *fg;
546         struct mlx5_flow_table *ft;
547         struct mlx5_core_dev *dev;
548
549         fs_get_obj(fg, node);
550         fs_get_obj(ft, fg->node.parent);
551         dev = get_dev(&ft->node);
552         trace_mlx5_fs_del_fg(fg);
553
554         if (fg->node.active && mlx5_cmd_destroy_flow_group(dev, ft, fg->id))
555                 mlx5_core_warn(dev, "flow steering can't destroy fg %d of ft %d\n",
556                                fg->id, ft->id);
557 }
558
559 static void del_sw_flow_group(struct fs_node *node)
560 {
561         struct mlx5_flow_steering *steering = get_steering(node);
562         struct mlx5_flow_group *fg;
563         struct mlx5_flow_table *ft;
564         int err;
565
566         fs_get_obj(fg, node);
567         fs_get_obj(ft, fg->node.parent);
568
569         rhashtable_destroy(&fg->ftes_hash);
570         ida_destroy(&fg->fte_allocator);
571         if (ft->autogroup.active)
572                 ft->autogroup.num_groups--;
573         err = rhltable_remove(&ft->fgs_hash,
574                               &fg->hash,
575                               rhash_fg);
576         WARN_ON(err);
577         kmem_cache_free(steering->fgs_cache, fg);
578 }
579
580 static int insert_fte(struct mlx5_flow_group *fg, struct fs_fte *fte)
581 {
582         int index;
583         int ret;
584
585         index = ida_simple_get(&fg->fte_allocator, 0, fg->max_ftes, GFP_KERNEL);
586         if (index < 0)
587                 return index;
588
589         fte->index = index + fg->start_index;
590         ret = rhashtable_insert_fast(&fg->ftes_hash,
591                                      &fte->hash,
592                                      rhash_fte);
593         if (ret)
594                 goto err_ida_remove;
595
596         tree_add_node(&fte->node, &fg->node);
597         list_add_tail(&fte->node.list, &fg->node.children);
598         return 0;
599
600 err_ida_remove:
601         ida_simple_remove(&fg->fte_allocator, index);
602         return ret;
603 }
604
605 static struct fs_fte *alloc_fte(struct mlx5_flow_table *ft,
606                                 u32 *match_value,
607                                 struct mlx5_flow_act *flow_act)
608 {
609         struct mlx5_flow_steering *steering = get_steering(&ft->node);
610         struct fs_fte *fte;
611
612         fte = kmem_cache_zalloc(steering->ftes_cache, GFP_KERNEL);
613         if (!fte)
614                 return ERR_PTR(-ENOMEM);
615
616         memcpy(fte->val, match_value, sizeof(fte->val));
617         fte->node.type =  FS_TYPE_FLOW_ENTRY;
618         fte->flow_tag = flow_act->flow_tag;
619         fte->action = flow_act->action;
620         fte->encap_id = flow_act->encap_id;
621         fte->modify_id = flow_act->modify_id;
622
623         tree_init_node(&fte->node, del_hw_fte, del_sw_fte);
624
625         return fte;
626 }
627
628 static void dealloc_flow_group(struct mlx5_flow_steering *steering,
629                                struct mlx5_flow_group *fg)
630 {
631         rhashtable_destroy(&fg->ftes_hash);
632         kmem_cache_free(steering->fgs_cache, fg);
633 }
634
635 static struct mlx5_flow_group *alloc_flow_group(struct mlx5_flow_steering *steering,
636                                                 u8 match_criteria_enable,
637                                                 void *match_criteria,
638                                                 int start_index,
639                                                 int end_index)
640 {
641         struct mlx5_flow_group *fg;
642         int ret;
643
644         fg = kmem_cache_zalloc(steering->fgs_cache, GFP_KERNEL);
645         if (!fg)
646                 return ERR_PTR(-ENOMEM);
647
648         ret = rhashtable_init(&fg->ftes_hash, &rhash_fte);
649         if (ret) {
650                 kmem_cache_free(steering->fgs_cache, fg);
651                 return ERR_PTR(ret);
652 }
653         ida_init(&fg->fte_allocator);
654         fg->mask.match_criteria_enable = match_criteria_enable;
655         memcpy(&fg->mask.match_criteria, match_criteria,
656                sizeof(fg->mask.match_criteria));
657         fg->node.type =  FS_TYPE_FLOW_GROUP;
658         fg->start_index = start_index;
659         fg->max_ftes = end_index - start_index + 1;
660
661         return fg;
662 }
663
664 static struct mlx5_flow_group *alloc_insert_flow_group(struct mlx5_flow_table *ft,
665                                                        u8 match_criteria_enable,
666                                                        void *match_criteria,
667                                                        int start_index,
668                                                        int end_index,
669                                                        struct list_head *prev)
670 {
671         struct mlx5_flow_steering *steering = get_steering(&ft->node);
672         struct mlx5_flow_group *fg;
673         int ret;
674
675         fg = alloc_flow_group(steering, match_criteria_enable, match_criteria,
676                               start_index, end_index);
677         if (IS_ERR(fg))
678                 return fg;
679
680         /* initialize refcnt, add to parent list */
681         ret = rhltable_insert(&ft->fgs_hash,
682                               &fg->hash,
683                               rhash_fg);
684         if (ret) {
685                 dealloc_flow_group(steering, fg);
686                 return ERR_PTR(ret);
687         }
688
689         tree_init_node(&fg->node, del_hw_flow_group, del_sw_flow_group);
690         tree_add_node(&fg->node, &ft->node);
691         /* Add node to group list */
692         list_add(&fg->node.list, prev);
693         atomic_inc(&ft->node.version);
694
695         return fg;
696 }
697
698 static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport, int max_fte,
699                                                 enum fs_flow_table_type table_type,
700                                                 enum fs_flow_table_op_mod op_mod,
701                                                 u32 flags)
702 {
703         struct mlx5_flow_table *ft;
704         int ret;
705
706         ft  = kzalloc(sizeof(*ft), GFP_KERNEL);
707         if (!ft)
708                 return ERR_PTR(-ENOMEM);
709
710         ret = rhltable_init(&ft->fgs_hash, &rhash_fg);
711         if (ret) {
712                 kfree(ft);
713                 return ERR_PTR(ret);
714         }
715
716         ft->level = level;
717         ft->node.type = FS_TYPE_FLOW_TABLE;
718         ft->op_mod = op_mod;
719         ft->type = table_type;
720         ft->vport = vport;
721         ft->max_fte = max_fte;
722         ft->flags = flags;
723         INIT_LIST_HEAD(&ft->fwd_rules);
724         mutex_init(&ft->lock);
725
726         return ft;
727 }
728
729 /* If reverse is false, then we search for the first flow table in the
730  * root sub-tree from start(closest from right), else we search for the
731  * last flow table in the root sub-tree till start(closest from left).
732  */
733 static struct mlx5_flow_table *find_closest_ft_recursive(struct fs_node  *root,
734                                                          struct list_head *start,
735                                                          bool reverse)
736 {
737 #define list_advance_entry(pos, reverse)                \
738         ((reverse) ? list_prev_entry(pos, list) : list_next_entry(pos, list))
739
740 #define list_for_each_advance_continue(pos, head, reverse)      \
741         for (pos = list_advance_entry(pos, reverse);            \
742              &pos->list != (head);                              \
743              pos = list_advance_entry(pos, reverse))
744
745         struct fs_node *iter = list_entry(start, struct fs_node, list);
746         struct mlx5_flow_table *ft = NULL;
747
748         if (!root)
749                 return NULL;
750
751         list_for_each_advance_continue(iter, &root->children, reverse) {
752                 if (iter->type == FS_TYPE_FLOW_TABLE) {
753                         fs_get_obj(ft, iter);
754                         return ft;
755                 }
756                 ft = find_closest_ft_recursive(iter, &iter->children, reverse);
757                 if (ft)
758                         return ft;
759         }
760
761         return ft;
762 }
763
764 /* If reverse if false then return the first flow table in next priority of
765  * prio in the tree, else return the last flow table in the previous priority
766  * of prio in the tree.
767  */
768 static struct mlx5_flow_table *find_closest_ft(struct fs_prio *prio, bool reverse)
769 {
770         struct mlx5_flow_table *ft = NULL;
771         struct fs_node *curr_node;
772         struct fs_node *parent;
773
774         parent = prio->node.parent;
775         curr_node = &prio->node;
776         while (!ft && parent) {
777                 ft = find_closest_ft_recursive(parent, &curr_node->list, reverse);
778                 curr_node = parent;
779                 parent = curr_node->parent;
780         }
781         return ft;
782 }
783
784 /* Assuming all the tree is locked by mutex chain lock */
785 static struct mlx5_flow_table *find_next_chained_ft(struct fs_prio *prio)
786 {
787         return find_closest_ft(prio, false);
788 }
789
790 /* Assuming all the tree is locked by mutex chain lock */
791 static struct mlx5_flow_table *find_prev_chained_ft(struct fs_prio *prio)
792 {
793         return find_closest_ft(prio, true);
794 }
795
796 static int connect_fts_in_prio(struct mlx5_core_dev *dev,
797                                struct fs_prio *prio,
798                                struct mlx5_flow_table *ft)
799 {
800         struct mlx5_flow_table *iter;
801         int i = 0;
802         int err;
803
804         fs_for_each_ft(iter, prio) {
805                 i++;
806                 err = mlx5_cmd_modify_flow_table(dev,
807                                                  iter,
808                                                  ft);
809                 if (err) {
810                         mlx5_core_warn(dev, "Failed to modify flow table %d\n",
811                                        iter->id);
812                         /* The driver is out of sync with the FW */
813                         if (i > 1)
814                                 WARN_ON(true);
815                         return err;
816                 }
817         }
818         return 0;
819 }
820
821 /* Connect flow tables from previous priority of prio to ft */
822 static int connect_prev_fts(struct mlx5_core_dev *dev,
823                             struct mlx5_flow_table *ft,
824                             struct fs_prio *prio)
825 {
826         struct mlx5_flow_table *prev_ft;
827
828         prev_ft = find_prev_chained_ft(prio);
829         if (prev_ft) {
830                 struct fs_prio *prev_prio;
831
832                 fs_get_obj(prev_prio, prev_ft->node.parent);
833                 return connect_fts_in_prio(dev, prev_prio, ft);
834         }
835         return 0;
836 }
837
838 static int update_root_ft_create(struct mlx5_flow_table *ft, struct fs_prio
839                                  *prio)
840 {
841         struct mlx5_flow_root_namespace *root = find_root(&prio->node);
842         struct mlx5_ft_underlay_qp *uqp;
843         int min_level = INT_MAX;
844         int err;
845         u32 qpn;
846
847         if (root->root_ft)
848                 min_level = root->root_ft->level;
849
850         if (ft->level >= min_level)
851                 return 0;
852
853         if (list_empty(&root->underlay_qpns)) {
854                 /* Don't set any QPN (zero) in case QPN list is empty */
855                 qpn = 0;
856                 err = mlx5_cmd_update_root_ft(root->dev, ft, qpn, false);
857         } else {
858                 list_for_each_entry(uqp, &root->underlay_qpns, list) {
859                         qpn = uqp->qpn;
860                         err = mlx5_cmd_update_root_ft(root->dev, ft, qpn,
861                                                       false);
862                         if (err)
863                                 break;
864                 }
865         }
866
867         if (err)
868                 mlx5_core_warn(root->dev,
869                                "Update root flow table of id(%u) qpn(%d) failed\n",
870                                ft->id, qpn);
871         else
872                 root->root_ft = ft;
873
874         return err;
875 }
876
877 static int _mlx5_modify_rule_destination(struct mlx5_flow_rule *rule,
878                                          struct mlx5_flow_destination *dest)
879 {
880         struct mlx5_flow_table *ft;
881         struct mlx5_flow_group *fg;
882         struct fs_fte *fte;
883         int modify_mask = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST);
884         int err = 0;
885
886         fs_get_obj(fte, rule->node.parent);
887         if (!(fte->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST))
888                 return -EINVAL;
889         down_write_ref_node(&fte->node);
890         fs_get_obj(fg, fte->node.parent);
891         fs_get_obj(ft, fg->node.parent);
892
893         memcpy(&rule->dest_attr, dest, sizeof(*dest));
894         err = mlx5_cmd_update_fte(get_dev(&ft->node),
895                                   ft, fg->id,
896                                   modify_mask,
897                                   fte);
898         up_write_ref_node(&fte->node);
899
900         return err;
901 }
902
903 int mlx5_modify_rule_destination(struct mlx5_flow_handle *handle,
904                                  struct mlx5_flow_destination *new_dest,
905                                  struct mlx5_flow_destination *old_dest)
906 {
907         int i;
908
909         if (!old_dest) {
910                 if (handle->num_rules != 1)
911                         return -EINVAL;
912                 return _mlx5_modify_rule_destination(handle->rule[0],
913                                                      new_dest);
914         }
915
916         for (i = 0; i < handle->num_rules; i++) {
917                 if (mlx5_flow_dests_cmp(new_dest, &handle->rule[i]->dest_attr))
918                         return _mlx5_modify_rule_destination(handle->rule[i],
919                                                              new_dest);
920         }
921
922         return -EINVAL;
923 }
924
925 /* Modify/set FWD rules that point on old_next_ft to point on new_next_ft  */
926 static int connect_fwd_rules(struct mlx5_core_dev *dev,
927                              struct mlx5_flow_table *new_next_ft,
928                              struct mlx5_flow_table *old_next_ft)
929 {
930         struct mlx5_flow_destination dest = {};
931         struct mlx5_flow_rule *iter;
932         int err = 0;
933
934         /* new_next_ft and old_next_ft could be NULL only
935          * when we create/destroy the anchor flow table.
936          */
937         if (!new_next_ft || !old_next_ft)
938                 return 0;
939
940         dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
941         dest.ft = new_next_ft;
942
943         mutex_lock(&old_next_ft->lock);
944         list_splice_init(&old_next_ft->fwd_rules, &new_next_ft->fwd_rules);
945         mutex_unlock(&old_next_ft->lock);
946         list_for_each_entry(iter, &new_next_ft->fwd_rules, next_ft) {
947                 err = _mlx5_modify_rule_destination(iter, &dest);
948                 if (err)
949                         pr_err("mlx5_core: failed to modify rule to point on flow table %d\n",
950                                new_next_ft->id);
951         }
952         return 0;
953 }
954
955 static int connect_flow_table(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft,
956                               struct fs_prio *prio)
957 {
958         struct mlx5_flow_table *next_ft;
959         int err = 0;
960
961         /* Connect_prev_fts and update_root_ft_create are mutually exclusive */
962
963         if (list_empty(&prio->node.children)) {
964                 err = connect_prev_fts(dev, ft, prio);
965                 if (err)
966                         return err;
967
968                 next_ft = find_next_chained_ft(prio);
969                 err = connect_fwd_rules(dev, ft, next_ft);
970                 if (err)
971                         return err;
972         }
973
974         if (MLX5_CAP_FLOWTABLE(dev,
975                                flow_table_properties_nic_receive.modify_root))
976                 err = update_root_ft_create(ft, prio);
977         return err;
978 }
979
980 static void list_add_flow_table(struct mlx5_flow_table *ft,
981                                 struct fs_prio *prio)
982 {
983         struct list_head *prev = &prio->node.children;
984         struct mlx5_flow_table *iter;
985
986         fs_for_each_ft(iter, prio) {
987                 if (iter->level > ft->level)
988                         break;
989                 prev = &iter->node.list;
990         }
991         list_add(&ft->node.list, prev);
992 }
993
994 static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
995                                                         struct mlx5_flow_table_attr *ft_attr,
996                                                         enum fs_flow_table_op_mod op_mod,
997                                                         u16 vport)
998 {
999         struct mlx5_flow_root_namespace *root = find_root(&ns->node);
1000         struct mlx5_flow_table *next_ft = NULL;
1001         struct fs_prio *fs_prio = NULL;
1002         struct mlx5_flow_table *ft;
1003         int log_table_sz;
1004         int err;
1005
1006         if (!root) {
1007                 pr_err("mlx5: flow steering failed to find root of namespace\n");
1008                 return ERR_PTR(-ENODEV);
1009         }
1010
1011         mutex_lock(&root->chain_lock);
1012         fs_prio = find_prio(ns, ft_attr->prio);
1013         if (!fs_prio) {
1014                 err = -EINVAL;
1015                 goto unlock_root;
1016         }
1017         if (ft_attr->level >= fs_prio->num_levels) {
1018                 err = -ENOSPC;
1019                 goto unlock_root;
1020         }
1021         /* The level is related to the
1022          * priority level range.
1023          */
1024         ft_attr->level += fs_prio->start_level;
1025         ft = alloc_flow_table(ft_attr->level,
1026                               vport,
1027                               ft_attr->max_fte ? roundup_pow_of_two(ft_attr->max_fte) : 0,
1028                               root->table_type,
1029                               op_mod, ft_attr->flags);
1030         if (IS_ERR(ft)) {
1031                 err = PTR_ERR(ft);
1032                 goto unlock_root;
1033         }
1034
1035         tree_init_node(&ft->node, del_hw_flow_table, del_sw_flow_table);
1036         log_table_sz = ft->max_fte ? ilog2(ft->max_fte) : 0;
1037         next_ft = find_next_chained_ft(fs_prio);
1038         err = mlx5_cmd_create_flow_table(root->dev, ft->vport, ft->op_mod, ft->type,
1039                                          ft->level, log_table_sz, next_ft, &ft->id,
1040                                          ft->flags);
1041         if (err)
1042                 goto free_ft;
1043
1044         err = connect_flow_table(root->dev, ft, fs_prio);
1045         if (err)
1046                 goto destroy_ft;
1047         ft->node.active = true;
1048         down_write_ref_node(&fs_prio->node);
1049         tree_add_node(&ft->node, &fs_prio->node);
1050         list_add_flow_table(ft, fs_prio);
1051         fs_prio->num_ft++;
1052         up_write_ref_node(&fs_prio->node);
1053         mutex_unlock(&root->chain_lock);
1054         return ft;
1055 destroy_ft:
1056         mlx5_cmd_destroy_flow_table(root->dev, ft);
1057 free_ft:
1058         kfree(ft);
1059 unlock_root:
1060         mutex_unlock(&root->chain_lock);
1061         return ERR_PTR(err);
1062 }
1063
1064 struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
1065                                                struct mlx5_flow_table_attr *ft_attr)
1066 {
1067         return __mlx5_create_flow_table(ns, ft_attr, FS_FT_OP_MOD_NORMAL, 0);
1068 }
1069
1070 struct mlx5_flow_table *mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
1071                                                      int prio, int max_fte,
1072                                                      u32 level, u16 vport)
1073 {
1074         struct mlx5_flow_table_attr ft_attr = {};
1075
1076         ft_attr.max_fte = max_fte;
1077         ft_attr.level   = level;
1078         ft_attr.prio    = prio;
1079
1080         return __mlx5_create_flow_table(ns, &ft_attr, FS_FT_OP_MOD_NORMAL, vport);
1081 }
1082
1083 struct mlx5_flow_table*
1084 mlx5_create_lag_demux_flow_table(struct mlx5_flow_namespace *ns,
1085                                  int prio, u32 level)
1086 {
1087         struct mlx5_flow_table_attr ft_attr = {};
1088
1089         ft_attr.level = level;
1090         ft_attr.prio  = prio;
1091         return __mlx5_create_flow_table(ns, &ft_attr, FS_FT_OP_MOD_LAG_DEMUX, 0);
1092 }
1093 EXPORT_SYMBOL(mlx5_create_lag_demux_flow_table);
1094
1095 struct mlx5_flow_table*
1096 mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
1097                                     int prio,
1098                                     int num_flow_table_entries,
1099                                     int max_num_groups,
1100                                     u32 level,
1101                                     u32 flags)
1102 {
1103         struct mlx5_flow_table_attr ft_attr = {};
1104         struct mlx5_flow_table *ft;
1105
1106         if (max_num_groups > num_flow_table_entries)
1107                 return ERR_PTR(-EINVAL);
1108
1109         ft_attr.max_fte = num_flow_table_entries;
1110         ft_attr.prio    = prio;
1111         ft_attr.level   = level;
1112         ft_attr.flags   = flags;
1113
1114         ft = mlx5_create_flow_table(ns, &ft_attr);
1115         if (IS_ERR(ft))
1116                 return ft;
1117
1118         ft->autogroup.active = true;
1119         ft->autogroup.required_groups = max_num_groups;
1120
1121         return ft;
1122 }
1123 EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table);
1124
1125 struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft,
1126                                                u32 *fg_in)
1127 {
1128         void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1129                                             fg_in, match_criteria);
1130         u8 match_criteria_enable = MLX5_GET(create_flow_group_in,
1131                                             fg_in,
1132                                             match_criteria_enable);
1133         int start_index = MLX5_GET(create_flow_group_in, fg_in,
1134                                    start_flow_index);
1135         int end_index = MLX5_GET(create_flow_group_in, fg_in,
1136                                  end_flow_index);
1137         struct mlx5_core_dev *dev = get_dev(&ft->node);
1138         struct mlx5_flow_group *fg;
1139         int err;
1140
1141         if (!check_valid_mask(match_criteria_enable, match_criteria))
1142                 return ERR_PTR(-EINVAL);
1143
1144         if (ft->autogroup.active)
1145                 return ERR_PTR(-EPERM);
1146
1147         down_write_ref_node(&ft->node);
1148         fg = alloc_insert_flow_group(ft, match_criteria_enable, match_criteria,
1149                                      start_index, end_index,
1150                                      ft->node.children.prev);
1151         up_write_ref_node(&ft->node);
1152         if (IS_ERR(fg))
1153                 return fg;
1154
1155         err = mlx5_cmd_create_flow_group(dev, ft, fg_in, &fg->id);
1156         if (err) {
1157                 tree_put_node(&fg->node);
1158                 return ERR_PTR(err);
1159         }
1160         trace_mlx5_fs_add_fg(fg);
1161         fg->node.active = true;
1162
1163         return fg;
1164 }
1165
1166 static struct mlx5_flow_rule *alloc_rule(struct mlx5_flow_destination *dest)
1167 {
1168         struct mlx5_flow_rule *rule;
1169
1170         rule = kzalloc(sizeof(*rule), GFP_KERNEL);
1171         if (!rule)
1172                 return NULL;
1173
1174         INIT_LIST_HEAD(&rule->next_ft);
1175         rule->node.type = FS_TYPE_FLOW_DEST;
1176         if (dest)
1177                 memcpy(&rule->dest_attr, dest, sizeof(*dest));
1178
1179         return rule;
1180 }
1181
1182 static struct mlx5_flow_handle *alloc_handle(int num_rules)
1183 {
1184         struct mlx5_flow_handle *handle;
1185
1186         handle = kzalloc(sizeof(*handle) + sizeof(handle->rule[0]) *
1187                           num_rules, GFP_KERNEL);
1188         if (!handle)
1189                 return NULL;
1190
1191         handle->num_rules = num_rules;
1192
1193         return handle;
1194 }
1195
1196 static void destroy_flow_handle(struct fs_fte *fte,
1197                                 struct mlx5_flow_handle *handle,
1198                                 struct mlx5_flow_destination *dest,
1199                                 int i)
1200 {
1201         for (; --i >= 0;) {
1202                 if (refcount_dec_and_test(&handle->rule[i]->node.refcount)) {
1203                         fte->dests_size--;
1204                         list_del(&handle->rule[i]->node.list);
1205                         kfree(handle->rule[i]);
1206                 }
1207         }
1208         kfree(handle);
1209 }
1210
1211 static struct mlx5_flow_handle *
1212 create_flow_handle(struct fs_fte *fte,
1213                    struct mlx5_flow_destination *dest,
1214                    int dest_num,
1215                    int *modify_mask,
1216                    bool *new_rule)
1217 {
1218         struct mlx5_flow_handle *handle;
1219         struct mlx5_flow_rule *rule = NULL;
1220         static int count = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS);
1221         static int dst = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST);
1222         int type;
1223         int i = 0;
1224
1225         handle = alloc_handle((dest_num) ? dest_num : 1);
1226         if (!handle)
1227                 return ERR_PTR(-ENOMEM);
1228
1229         do {
1230                 if (dest) {
1231                         rule = find_flow_rule(fte, dest + i);
1232                         if (rule) {
1233                                 refcount_inc(&rule->node.refcount);
1234                                 goto rule_found;
1235                         }
1236                 }
1237
1238                 *new_rule = true;
1239                 rule = alloc_rule(dest + i);
1240                 if (!rule)
1241                         goto free_rules;
1242
1243                 /* Add dest to dests list- we need flow tables to be in the
1244                  * end of the list for forward to next prio rules.
1245                  */
1246                 tree_init_node(&rule->node, NULL, del_sw_hw_rule);
1247                 if (dest &&
1248                     dest[i].type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
1249                         list_add(&rule->node.list, &fte->node.children);
1250                 else
1251                         list_add_tail(&rule->node.list, &fte->node.children);
1252                 if (dest) {
1253                         fte->dests_size++;
1254
1255                         type = dest[i].type ==
1256                                 MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1257                         *modify_mask |= type ? count : dst;
1258                 }
1259 rule_found:
1260                 handle->rule[i] = rule;
1261         } while (++i < dest_num);
1262
1263         return handle;
1264
1265 free_rules:
1266         destroy_flow_handle(fte, handle, dest, i);
1267         return ERR_PTR(-ENOMEM);
1268 }
1269
1270 /* fte should not be deleted while calling this function */
1271 static struct mlx5_flow_handle *
1272 add_rule_fte(struct fs_fte *fte,
1273              struct mlx5_flow_group *fg,
1274              struct mlx5_flow_destination *dest,
1275              int dest_num,
1276              bool update_action)
1277 {
1278         struct mlx5_flow_handle *handle;
1279         struct mlx5_flow_table *ft;
1280         int modify_mask = 0;
1281         int err;
1282         bool new_rule = false;
1283
1284         handle = create_flow_handle(fte, dest, dest_num, &modify_mask,
1285                                     &new_rule);
1286         if (IS_ERR(handle) || !new_rule)
1287                 goto out;
1288
1289         if (update_action)
1290                 modify_mask |= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION);
1291
1292         fs_get_obj(ft, fg->node.parent);
1293         if (!(fte->status & FS_FTE_STATUS_EXISTING))
1294                 err = mlx5_cmd_create_fte(get_dev(&ft->node),
1295                                           ft, fg->id, fte);
1296         else
1297                 err = mlx5_cmd_update_fte(get_dev(&ft->node),
1298                                           ft, fg->id, modify_mask, fte);
1299         if (err)
1300                 goto free_handle;
1301
1302         fte->node.active = true;
1303         fte->status |= FS_FTE_STATUS_EXISTING;
1304         atomic_inc(&fte->node.version);
1305
1306 out:
1307         return handle;
1308
1309 free_handle:
1310         destroy_flow_handle(fte, handle, dest, handle->num_rules);
1311         return ERR_PTR(err);
1312 }
1313
1314 static struct mlx5_flow_group *alloc_auto_flow_group(struct mlx5_flow_table  *ft,
1315                                                      struct mlx5_flow_spec *spec)
1316 {
1317         struct list_head *prev = &ft->node.children;
1318         struct mlx5_flow_group *fg;
1319         unsigned int candidate_index = 0;
1320         unsigned int group_size = 0;
1321
1322         if (!ft->autogroup.active)
1323                 return ERR_PTR(-ENOENT);
1324
1325         if (ft->autogroup.num_groups < ft->autogroup.required_groups)
1326                 /* We save place for flow groups in addition to max types */
1327                 group_size = ft->max_fte / (ft->autogroup.required_groups + 1);
1328
1329         /*  ft->max_fte == ft->autogroup.max_types */
1330         if (group_size == 0)
1331                 group_size = 1;
1332
1333         /* sorted by start_index */
1334         fs_for_each_fg(fg, ft) {
1335                 if (candidate_index + group_size > fg->start_index)
1336                         candidate_index = fg->start_index + fg->max_ftes;
1337                 else
1338                         break;
1339                 prev = &fg->node.list;
1340         }
1341
1342         if (candidate_index + group_size > ft->max_fte)
1343                 return ERR_PTR(-ENOSPC);
1344
1345         fg = alloc_insert_flow_group(ft,
1346                                      spec->match_criteria_enable,
1347                                      spec->match_criteria,
1348                                      candidate_index,
1349                                      candidate_index + group_size - 1,
1350                                      prev);
1351         if (IS_ERR(fg))
1352                 goto out;
1353
1354         ft->autogroup.num_groups++;
1355
1356 out:
1357         return fg;
1358 }
1359
1360 static int create_auto_flow_group(struct mlx5_flow_table *ft,
1361                                   struct mlx5_flow_group *fg)
1362 {
1363         struct mlx5_core_dev *dev = get_dev(&ft->node);
1364         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1365         void *match_criteria_addr;
1366         int err;
1367         u32 *in;
1368
1369         in = kvzalloc(inlen, GFP_KERNEL);
1370         if (!in)
1371                 return -ENOMEM;
1372
1373         MLX5_SET(create_flow_group_in, in, match_criteria_enable,
1374                  fg->mask.match_criteria_enable);
1375         MLX5_SET(create_flow_group_in, in, start_flow_index, fg->start_index);
1376         MLX5_SET(create_flow_group_in, in, end_flow_index,   fg->start_index +
1377                  fg->max_ftes - 1);
1378         match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in,
1379                                            in, match_criteria);
1380         memcpy(match_criteria_addr, fg->mask.match_criteria,
1381                sizeof(fg->mask.match_criteria));
1382
1383         err = mlx5_cmd_create_flow_group(dev, ft, in, &fg->id);
1384         if (!err) {
1385                 fg->node.active = true;
1386                 trace_mlx5_fs_add_fg(fg);
1387         }
1388
1389         kvfree(in);
1390         return err;
1391 }
1392
1393 static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1,
1394                                 struct mlx5_flow_destination *d2)
1395 {
1396         if (d1->type == d2->type) {
1397                 if ((d1->type == MLX5_FLOW_DESTINATION_TYPE_VPORT &&
1398                      d1->vport_num == d2->vport_num) ||
1399                     (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE &&
1400                      d1->ft == d2->ft) ||
1401                     (d1->type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
1402                      d1->tir_num == d2->tir_num))
1403                         return true;
1404         }
1405
1406         return false;
1407 }
1408
1409 static struct mlx5_flow_rule *find_flow_rule(struct fs_fte *fte,
1410                                              struct mlx5_flow_destination *dest)
1411 {
1412         struct mlx5_flow_rule *rule;
1413
1414         list_for_each_entry(rule, &fte->node.children, node.list) {
1415                 if (mlx5_flow_dests_cmp(&rule->dest_attr, dest))
1416                         return rule;
1417         }
1418         return NULL;
1419 }
1420
1421 static bool check_conflicting_actions(u32 action1, u32 action2)
1422 {
1423         u32 xored_actions = action1 ^ action2;
1424
1425         /* if one rule only wants to count, it's ok */
1426         if (action1 == MLX5_FLOW_CONTEXT_ACTION_COUNT ||
1427             action2 == MLX5_FLOW_CONTEXT_ACTION_COUNT)
1428                 return false;
1429
1430         if (xored_actions & (MLX5_FLOW_CONTEXT_ACTION_DROP  |
1431                              MLX5_FLOW_CONTEXT_ACTION_ENCAP |
1432                              MLX5_FLOW_CONTEXT_ACTION_DECAP))
1433                 return true;
1434
1435         return false;
1436 }
1437
1438 static int check_conflicting_ftes(struct fs_fte *fte, const struct mlx5_flow_act *flow_act)
1439 {
1440         if (check_conflicting_actions(flow_act->action, fte->action)) {
1441                 mlx5_core_warn(get_dev(&fte->node),
1442                                "Found two FTEs with conflicting actions\n");
1443                 return -EEXIST;
1444         }
1445
1446         if (fte->flow_tag != flow_act->flow_tag) {
1447                 mlx5_core_warn(get_dev(&fte->node),
1448                                "FTE flow tag %u already exists with different flow tag %u\n",
1449                                fte->flow_tag,
1450                                flow_act->flow_tag);
1451                 return -EEXIST;
1452         }
1453
1454         return 0;
1455 }
1456
1457 static struct mlx5_flow_handle *add_rule_fg(struct mlx5_flow_group *fg,
1458                                             u32 *match_value,
1459                                             struct mlx5_flow_act *flow_act,
1460                                             struct mlx5_flow_destination *dest,
1461                                             int dest_num,
1462                                             struct fs_fte *fte)
1463 {
1464         struct mlx5_flow_handle *handle;
1465         int old_action;
1466         int i;
1467         int ret;
1468
1469         ret = check_conflicting_ftes(fte, flow_act);
1470         if (ret)
1471                 return ERR_PTR(ret);
1472
1473         old_action = fte->action;
1474         fte->action |= flow_act->action;
1475         handle = add_rule_fte(fte, fg, dest, dest_num,
1476                               old_action != flow_act->action);
1477         if (IS_ERR(handle)) {
1478                 fte->action = old_action;
1479                 return handle;
1480         }
1481         trace_mlx5_fs_set_fte(fte, false);
1482
1483         for (i = 0; i < handle->num_rules; i++) {
1484                 if (refcount_read(&handle->rule[i]->node.refcount) == 1) {
1485                         tree_add_node(&handle->rule[i]->node, &fte->node);
1486                         trace_mlx5_fs_add_rule(handle->rule[i]);
1487                 }
1488         }
1489         return handle;
1490 }
1491
1492 struct mlx5_fc *mlx5_flow_rule_counter(struct mlx5_flow_handle *handle)
1493 {
1494         struct mlx5_flow_rule *dst;
1495         struct fs_fte *fte;
1496
1497         fs_get_obj(fte, handle->rule[0]->node.parent);
1498
1499         fs_for_each_dst(dst, fte) {
1500                 if (dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_COUNTER)
1501                         return dst->dest_attr.counter;
1502         }
1503
1504         return NULL;
1505 }
1506
1507 static bool counter_is_valid(struct mlx5_fc *counter, u32 action)
1508 {
1509         if (!(action & MLX5_FLOW_CONTEXT_ACTION_COUNT))
1510                 return !counter;
1511
1512         if (!counter)
1513                 return false;
1514
1515         return (action & (MLX5_FLOW_CONTEXT_ACTION_DROP |
1516                           MLX5_FLOW_CONTEXT_ACTION_FWD_DEST));
1517 }
1518
1519 static bool dest_is_valid(struct mlx5_flow_destination *dest,
1520                           u32 action,
1521                           struct mlx5_flow_table *ft)
1522 {
1523         if (dest && (dest->type == MLX5_FLOW_DESTINATION_TYPE_COUNTER))
1524                 return counter_is_valid(dest->counter, action);
1525
1526         if (!(action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST))
1527                 return true;
1528
1529         if (!dest || ((dest->type ==
1530             MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) &&
1531             (dest->ft->level <= ft->level)))
1532                 return false;
1533         return true;
1534 }
1535
1536 struct match_list {
1537         struct list_head        list;
1538         struct mlx5_flow_group *g;
1539 };
1540
1541 struct match_list_head {
1542         struct list_head  list;
1543         struct match_list first;
1544 };
1545
1546 static void free_match_list(struct match_list_head *head)
1547 {
1548         if (!list_empty(&head->list)) {
1549                 struct match_list *iter, *match_tmp;
1550
1551                 list_del(&head->first.list);
1552                 tree_put_node(&head->first.g->node);
1553                 list_for_each_entry_safe(iter, match_tmp, &head->list,
1554                                          list) {
1555                         tree_put_node(&iter->g->node);
1556                         list_del(&iter->list);
1557                         kfree(iter);
1558                 }
1559         }
1560 }
1561
1562 static int build_match_list(struct match_list_head *match_head,
1563                             struct mlx5_flow_table *ft,
1564                             struct mlx5_flow_spec *spec)
1565 {
1566         struct rhlist_head *tmp, *list;
1567         struct mlx5_flow_group *g;
1568         int err = 0;
1569
1570         rcu_read_lock();
1571         INIT_LIST_HEAD(&match_head->list);
1572         /* Collect all fgs which has a matching match_criteria */
1573         list = rhltable_lookup(&ft->fgs_hash, spec, rhash_fg);
1574         /* RCU is atomic, we can't execute FW commands here */
1575         rhl_for_each_entry_rcu(g, tmp, list, hash) {
1576                 struct match_list *curr_match;
1577
1578                 if (likely(list_empty(&match_head->list))) {
1579                         if (!tree_get_node(&g->node))
1580                                 continue;
1581                         match_head->first.g = g;
1582                         list_add_tail(&match_head->first.list,
1583                                       &match_head->list);
1584                         continue;
1585                 }
1586
1587                 curr_match = kmalloc(sizeof(*curr_match), GFP_ATOMIC);
1588                 if (!curr_match) {
1589                         free_match_list(match_head);
1590                         err = -ENOMEM;
1591                         goto out;
1592                 }
1593                 if (!tree_get_node(&g->node)) {
1594                         kfree(curr_match);
1595                         continue;
1596                 }
1597                 curr_match->g = g;
1598                 list_add_tail(&curr_match->list, &match_head->list);
1599         }
1600 out:
1601         rcu_read_unlock();
1602         return err;
1603 }
1604
1605 static u64 matched_fgs_get_version(struct list_head *match_head)
1606 {
1607         struct match_list *iter;
1608         u64 version = 0;
1609
1610         list_for_each_entry(iter, match_head, list)
1611                 version += (u64)atomic_read(&iter->g->node.version);
1612         return version;
1613 }
1614
1615 static struct mlx5_flow_handle *
1616 try_add_to_existing_fg(struct mlx5_flow_table *ft,
1617                        struct list_head *match_head,
1618                        struct mlx5_flow_spec *spec,
1619                        struct mlx5_flow_act *flow_act,
1620                        struct mlx5_flow_destination *dest,
1621                        int dest_num,
1622                        int ft_version)
1623 {
1624         struct mlx5_flow_steering *steering = get_steering(&ft->node);
1625         struct mlx5_flow_group *g;
1626         struct mlx5_flow_handle *rule;
1627         struct match_list *iter;
1628         bool take_write = false;
1629         struct fs_fte *fte;
1630         u64  version;
1631         int err;
1632
1633         fte = alloc_fte(ft, spec->match_value, flow_act);
1634         if (IS_ERR(fte))
1635                 return  ERR_PTR(-ENOMEM);
1636
1637         list_for_each_entry(iter, match_head, list) {
1638                 nested_down_read_ref_node(&iter->g->node, FS_LOCK_PARENT);
1639                 ida_pre_get(&iter->g->fte_allocator, GFP_KERNEL);
1640         }
1641
1642 search_again_locked:
1643         version = matched_fgs_get_version(match_head);
1644         /* Try to find a fg that already contains a matching fte */
1645         list_for_each_entry(iter, match_head, list) {
1646                 struct fs_fte *fte_tmp;
1647
1648                 g = iter->g;
1649                 fte_tmp = rhashtable_lookup_fast(&g->ftes_hash, spec->match_value,
1650                                                  rhash_fte);
1651                 if (!fte_tmp || !tree_get_node(&fte_tmp->node))
1652                         continue;
1653
1654                 nested_down_write_ref_node(&fte_tmp->node, FS_LOCK_CHILD);
1655                 if (!take_write) {
1656                         list_for_each_entry(iter, match_head, list)
1657                                 up_read_ref_node(&iter->g->node);
1658                 } else {
1659                         list_for_each_entry(iter, match_head, list)
1660                                 up_write_ref_node(&iter->g->node);
1661                 }
1662
1663                 rule = add_rule_fg(g, spec->match_value,
1664                                    flow_act, dest, dest_num, fte_tmp);
1665                 up_write_ref_node(&fte_tmp->node);
1666                 tree_put_node(&fte_tmp->node);
1667                 kmem_cache_free(steering->ftes_cache, fte);
1668                 return rule;
1669         }
1670
1671         /* No group with matching fte found. Try to add a new fte to any
1672          * matching fg.
1673          */
1674
1675         if (!take_write) {
1676                 list_for_each_entry(iter, match_head, list)
1677                         up_read_ref_node(&iter->g->node);
1678                 list_for_each_entry(iter, match_head, list)
1679                         nested_down_write_ref_node(&iter->g->node,
1680                                                    FS_LOCK_PARENT);
1681                 take_write = true;
1682         }
1683
1684         /* Check the ft version, for case that new flow group
1685          * was added while the fgs weren't locked
1686          */
1687         if (atomic_read(&ft->node.version) != ft_version) {
1688                 rule = ERR_PTR(-EAGAIN);
1689                 goto out;
1690         }
1691
1692         /* Check the fgs version, for case the new FTE with the
1693          * same values was added while the fgs weren't locked
1694          */
1695         if (version != matched_fgs_get_version(match_head))
1696                 goto search_again_locked;
1697
1698         list_for_each_entry(iter, match_head, list) {
1699                 g = iter->g;
1700
1701                 if (!g->node.active)
1702                         continue;
1703                 err = insert_fte(g, fte);
1704                 if (err) {
1705                         if (err == -ENOSPC)
1706                                 continue;
1707                         list_for_each_entry(iter, match_head, list)
1708                                 up_write_ref_node(&iter->g->node);
1709                         kmem_cache_free(steering->ftes_cache, fte);
1710                         return ERR_PTR(err);
1711                 }
1712
1713                 nested_down_write_ref_node(&fte->node, FS_LOCK_CHILD);
1714                 list_for_each_entry(iter, match_head, list)
1715                         up_write_ref_node(&iter->g->node);
1716                 rule = add_rule_fg(g, spec->match_value,
1717                                    flow_act, dest, dest_num, fte);
1718                 up_write_ref_node(&fte->node);
1719                 tree_put_node(&fte->node);
1720                 return rule;
1721         }
1722         rule = ERR_PTR(-ENOENT);
1723 out:
1724         list_for_each_entry(iter, match_head, list)
1725                 up_write_ref_node(&iter->g->node);
1726         kmem_cache_free(steering->ftes_cache, fte);
1727         return rule;
1728 }
1729
1730 static struct mlx5_flow_handle *
1731 _mlx5_add_flow_rules(struct mlx5_flow_table *ft,
1732                      struct mlx5_flow_spec *spec,
1733                      struct mlx5_flow_act *flow_act,
1734                      struct mlx5_flow_destination *dest,
1735                      int dest_num)
1736
1737 {
1738         struct mlx5_flow_steering *steering = get_steering(&ft->node);
1739         struct mlx5_flow_group *g;
1740         struct mlx5_flow_handle *rule;
1741         struct match_list_head match_head;
1742         bool take_write = false;
1743         struct fs_fte *fte;
1744         int version;
1745         int err;
1746         int i;
1747
1748         if (!check_valid_spec(spec))
1749                 return ERR_PTR(-EINVAL);
1750
1751         for (i = 0; i < dest_num; i++) {
1752                 if (!dest_is_valid(&dest[i], flow_act->action, ft))
1753                         return ERR_PTR(-EINVAL);
1754         }
1755         nested_down_read_ref_node(&ft->node, FS_LOCK_GRANDPARENT);
1756 search_again_locked:
1757         version = atomic_read(&ft->node.version);
1758
1759         /* Collect all fgs which has a matching match_criteria */
1760         err = build_match_list(&match_head, ft, spec);
1761         if (err)
1762                 return ERR_PTR(err);
1763
1764         if (!take_write)
1765                 up_read_ref_node(&ft->node);
1766
1767         rule = try_add_to_existing_fg(ft, &match_head.list, spec, flow_act, dest,
1768                                       dest_num, version);
1769         free_match_list(&match_head);
1770         if (!IS_ERR(rule) ||
1771             (PTR_ERR(rule) != -ENOENT && PTR_ERR(rule) != -EAGAIN))
1772                 return rule;
1773
1774         if (!take_write) {
1775                 nested_down_write_ref_node(&ft->node, FS_LOCK_GRANDPARENT);
1776                 take_write = true;
1777         }
1778
1779         if (PTR_ERR(rule) == -EAGAIN ||
1780             version != atomic_read(&ft->node.version))
1781                 goto search_again_locked;
1782
1783         g = alloc_auto_flow_group(ft, spec);
1784         if (IS_ERR(g)) {
1785                 rule = (void *)g;
1786                 up_write_ref_node(&ft->node);
1787                 return rule;
1788         }
1789
1790         nested_down_write_ref_node(&g->node, FS_LOCK_PARENT);
1791         up_write_ref_node(&ft->node);
1792
1793         err = create_auto_flow_group(ft, g);
1794         if (err)
1795                 goto err_release_fg;
1796
1797         fte = alloc_fte(ft, spec->match_value, flow_act);
1798         if (IS_ERR(fte)) {
1799                 err = PTR_ERR(fte);
1800                 goto err_release_fg;
1801         }
1802
1803         err = insert_fte(g, fte);
1804         if (err) {
1805                 kmem_cache_free(steering->ftes_cache, fte);
1806                 goto err_release_fg;
1807         }
1808
1809         nested_down_write_ref_node(&fte->node, FS_LOCK_CHILD);
1810         up_write_ref_node(&g->node);
1811         rule = add_rule_fg(g, spec->match_value, flow_act, dest,
1812                            dest_num, fte);
1813         up_write_ref_node(&fte->node);
1814         tree_put_node(&fte->node);
1815         tree_put_node(&g->node);
1816         return rule;
1817
1818 err_release_fg:
1819         up_write_ref_node(&g->node);
1820         tree_put_node(&g->node);
1821         return ERR_PTR(err);
1822 }
1823
1824 static bool fwd_next_prio_supported(struct mlx5_flow_table *ft)
1825 {
1826         return ((ft->type == FS_FT_NIC_RX) &&
1827                 (MLX5_CAP_FLOWTABLE(get_dev(&ft->node), nic_rx_multi_path_tirs)));
1828 }
1829
1830 struct mlx5_flow_handle *
1831 mlx5_add_flow_rules(struct mlx5_flow_table *ft,
1832                     struct mlx5_flow_spec *spec,
1833                     struct mlx5_flow_act *flow_act,
1834                     struct mlx5_flow_destination *dest,
1835                     int dest_num)
1836 {
1837         struct mlx5_flow_root_namespace *root = find_root(&ft->node);
1838         struct mlx5_flow_destination gen_dest = {};
1839         struct mlx5_flow_table *next_ft = NULL;
1840         struct mlx5_flow_handle *handle = NULL;
1841         u32 sw_action = flow_act->action;
1842         struct fs_prio *prio;
1843
1844         fs_get_obj(prio, ft->node.parent);
1845         if (flow_act->action == MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO) {
1846                 if (!fwd_next_prio_supported(ft))
1847                         return ERR_PTR(-EOPNOTSUPP);
1848                 if (dest)
1849                         return ERR_PTR(-EINVAL);
1850                 mutex_lock(&root->chain_lock);
1851                 next_ft = find_next_chained_ft(prio);
1852                 if (next_ft) {
1853                         gen_dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1854                         gen_dest.ft = next_ft;
1855                         dest = &gen_dest;
1856                         dest_num = 1;
1857                         flow_act->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1858                 } else {
1859                         mutex_unlock(&root->chain_lock);
1860                         return ERR_PTR(-EOPNOTSUPP);
1861                 }
1862         }
1863
1864         handle = _mlx5_add_flow_rules(ft, spec, flow_act, dest, dest_num);
1865
1866         if (sw_action == MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO) {
1867                 if (!IS_ERR_OR_NULL(handle) &&
1868                     (list_empty(&handle->rule[0]->next_ft))) {
1869                         mutex_lock(&next_ft->lock);
1870                         list_add(&handle->rule[0]->next_ft,
1871                                  &next_ft->fwd_rules);
1872                         mutex_unlock(&next_ft->lock);
1873                         handle->rule[0]->sw_action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
1874                 }
1875                 mutex_unlock(&root->chain_lock);
1876         }
1877         return handle;
1878 }
1879 EXPORT_SYMBOL(mlx5_add_flow_rules);
1880
1881 void mlx5_del_flow_rules(struct mlx5_flow_handle *handle)
1882 {
1883         int i;
1884
1885         for (i = handle->num_rules - 1; i >= 0; i--)
1886                 tree_remove_node(&handle->rule[i]->node);
1887         kfree(handle);
1888 }
1889 EXPORT_SYMBOL(mlx5_del_flow_rules);
1890
1891 /* Assuming prio->node.children(flow tables) is sorted by level */
1892 static struct mlx5_flow_table *find_next_ft(struct mlx5_flow_table *ft)
1893 {
1894         struct fs_prio *prio;
1895
1896         fs_get_obj(prio, ft->node.parent);
1897
1898         if (!list_is_last(&ft->node.list, &prio->node.children))
1899                 return list_next_entry(ft, node.list);
1900         return find_next_chained_ft(prio);
1901 }
1902
1903 static int update_root_ft_destroy(struct mlx5_flow_table *ft)
1904 {
1905         struct mlx5_flow_root_namespace *root = find_root(&ft->node);
1906         struct mlx5_ft_underlay_qp *uqp;
1907         struct mlx5_flow_table *new_root_ft = NULL;
1908         int err = 0;
1909         u32 qpn;
1910
1911         if (root->root_ft != ft)
1912                 return 0;
1913
1914         new_root_ft = find_next_ft(ft);
1915
1916         if (!new_root_ft) {
1917                 root->root_ft = NULL;
1918                 return 0;
1919         }
1920
1921         if (list_empty(&root->underlay_qpns)) {
1922                 /* Don't set any QPN (zero) in case QPN list is empty */
1923                 qpn = 0;
1924                 err = mlx5_cmd_update_root_ft(root->dev, new_root_ft, qpn,
1925                                               false);
1926         } else {
1927                 list_for_each_entry(uqp, &root->underlay_qpns, list) {
1928                         qpn = uqp->qpn;
1929                         err = mlx5_cmd_update_root_ft(root->dev, new_root_ft,
1930                                                       qpn, false);
1931                         if (err)
1932                                 break;
1933                 }
1934         }
1935
1936         if (err)
1937                 mlx5_core_warn(root->dev,
1938                                "Update root flow table of id(%u) qpn(%d) failed\n",
1939                                ft->id, qpn);
1940         else
1941                 root->root_ft = new_root_ft;
1942
1943         return 0;
1944 }
1945
1946 /* Connect flow table from previous priority to
1947  * the next flow table.
1948  */
1949 static int disconnect_flow_table(struct mlx5_flow_table *ft)
1950 {
1951         struct mlx5_core_dev *dev = get_dev(&ft->node);
1952         struct mlx5_flow_table *next_ft;
1953         struct fs_prio *prio;
1954         int err = 0;
1955
1956         err = update_root_ft_destroy(ft);
1957         if (err)
1958                 return err;
1959
1960         fs_get_obj(prio, ft->node.parent);
1961         if  (!(list_first_entry(&prio->node.children,
1962                                 struct mlx5_flow_table,
1963                                 node.list) == ft))
1964                 return 0;
1965
1966         next_ft = find_next_chained_ft(prio);
1967         err = connect_fwd_rules(dev, next_ft, ft);
1968         if (err)
1969                 return err;
1970
1971         err = connect_prev_fts(dev, next_ft, prio);
1972         if (err)
1973                 mlx5_core_warn(dev, "Failed to disconnect flow table %d\n",
1974                                ft->id);
1975         return err;
1976 }
1977
1978 int mlx5_destroy_flow_table(struct mlx5_flow_table *ft)
1979 {
1980         struct mlx5_flow_root_namespace *root = find_root(&ft->node);
1981         int err = 0;
1982
1983         mutex_lock(&root->chain_lock);
1984         err = disconnect_flow_table(ft);
1985         if (err) {
1986                 mutex_unlock(&root->chain_lock);
1987                 return err;
1988         }
1989         if (tree_remove_node(&ft->node))
1990                 mlx5_core_warn(get_dev(&ft->node), "Flow table %d wasn't destroyed, refcount > 1\n",
1991                                ft->id);
1992         mutex_unlock(&root->chain_lock);
1993
1994         return err;
1995 }
1996 EXPORT_SYMBOL(mlx5_destroy_flow_table);
1997
1998 void mlx5_destroy_flow_group(struct mlx5_flow_group *fg)
1999 {
2000         if (tree_remove_node(&fg->node))
2001                 mlx5_core_warn(get_dev(&fg->node), "Flow group %d wasn't destroyed, refcount > 1\n",
2002                                fg->id);
2003 }
2004
2005 struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
2006                                                     enum mlx5_flow_namespace_type type)
2007 {
2008         struct mlx5_flow_steering *steering = dev->priv.steering;
2009         struct mlx5_flow_root_namespace *root_ns;
2010         int prio;
2011         struct fs_prio *fs_prio;
2012         struct mlx5_flow_namespace *ns;
2013
2014         if (!steering)
2015                 return NULL;
2016
2017         switch (type) {
2018         case MLX5_FLOW_NAMESPACE_BYPASS:
2019         case MLX5_FLOW_NAMESPACE_LAG:
2020         case MLX5_FLOW_NAMESPACE_OFFLOADS:
2021         case MLX5_FLOW_NAMESPACE_ETHTOOL:
2022         case MLX5_FLOW_NAMESPACE_KERNEL:
2023         case MLX5_FLOW_NAMESPACE_LEFTOVERS:
2024         case MLX5_FLOW_NAMESPACE_ANCHOR:
2025                 prio = type;
2026                 break;
2027         case MLX5_FLOW_NAMESPACE_FDB:
2028                 if (steering->fdb_root_ns)
2029                         return &steering->fdb_root_ns->ns;
2030                 else
2031                         return NULL;
2032         case MLX5_FLOW_NAMESPACE_SNIFFER_RX:
2033                 if (steering->sniffer_rx_root_ns)
2034                         return &steering->sniffer_rx_root_ns->ns;
2035                 else
2036                         return NULL;
2037         case MLX5_FLOW_NAMESPACE_SNIFFER_TX:
2038                 if (steering->sniffer_tx_root_ns)
2039                         return &steering->sniffer_tx_root_ns->ns;
2040                 else
2041                         return NULL;
2042         default:
2043                 return NULL;
2044         }
2045
2046         root_ns = steering->root_ns;
2047         if (!root_ns)
2048                 return NULL;
2049
2050         fs_prio = find_prio(&root_ns->ns, prio);
2051         if (!fs_prio)
2052                 return NULL;
2053
2054         ns = list_first_entry(&fs_prio->node.children,
2055                               typeof(*ns),
2056                               node.list);
2057
2058         return ns;
2059 }
2060 EXPORT_SYMBOL(mlx5_get_flow_namespace);
2061
2062 struct mlx5_flow_namespace *mlx5_get_flow_vport_acl_namespace(struct mlx5_core_dev *dev,
2063                                                               enum mlx5_flow_namespace_type type,
2064                                                               int vport)
2065 {
2066         struct mlx5_flow_steering *steering = dev->priv.steering;
2067
2068         if (!steering || vport >= MLX5_TOTAL_VPORTS(dev))
2069                 return NULL;
2070
2071         switch (type) {
2072         case MLX5_FLOW_NAMESPACE_ESW_EGRESS:
2073                 if (steering->esw_egress_root_ns &&
2074                     steering->esw_egress_root_ns[vport])
2075                         return &steering->esw_egress_root_ns[vport]->ns;
2076                 else
2077                         return NULL;
2078         case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
2079                 if (steering->esw_ingress_root_ns &&
2080                     steering->esw_ingress_root_ns[vport])
2081                         return &steering->esw_ingress_root_ns[vport]->ns;
2082                 else
2083                         return NULL;
2084         default:
2085                 return NULL;
2086         }
2087 }
2088
2089 static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns,
2090                                       unsigned int prio, int num_levels)
2091 {
2092         struct fs_prio *fs_prio;
2093
2094         fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL);
2095         if (!fs_prio)
2096                 return ERR_PTR(-ENOMEM);
2097
2098         fs_prio->node.type = FS_TYPE_PRIO;
2099         tree_init_node(&fs_prio->node, NULL, del_sw_prio);
2100         tree_add_node(&fs_prio->node, &ns->node);
2101         fs_prio->num_levels = num_levels;
2102         fs_prio->prio = prio;
2103         list_add_tail(&fs_prio->node.list, &ns->node.children);
2104
2105         return fs_prio;
2106 }
2107
2108 static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace
2109                                                      *ns)
2110 {
2111         ns->node.type = FS_TYPE_NAMESPACE;
2112
2113         return ns;
2114 }
2115
2116 static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio)
2117 {
2118         struct mlx5_flow_namespace      *ns;
2119
2120         ns = kzalloc(sizeof(*ns), GFP_KERNEL);
2121         if (!ns)
2122                 return ERR_PTR(-ENOMEM);
2123
2124         fs_init_namespace(ns);
2125         tree_init_node(&ns->node, NULL, del_sw_ns);
2126         tree_add_node(&ns->node, &prio->node);
2127         list_add_tail(&ns->node.list, &prio->node.children);
2128
2129         return ns;
2130 }
2131
2132 static int create_leaf_prios(struct mlx5_flow_namespace *ns, int prio,
2133                              struct init_tree_node *prio_metadata)
2134 {
2135         struct fs_prio *fs_prio;
2136         int i;
2137
2138         for (i = 0; i < prio_metadata->num_leaf_prios; i++) {
2139                 fs_prio = fs_create_prio(ns, prio++, prio_metadata->num_levels);
2140                 if (IS_ERR(fs_prio))
2141                         return PTR_ERR(fs_prio);
2142         }
2143         return 0;
2144 }
2145
2146 #define FLOW_TABLE_BIT_SZ 1
2147 #define GET_FLOW_TABLE_CAP(dev, offset) \
2148         ((be32_to_cpu(*((__be32 *)(dev->caps.hca_cur[MLX5_CAP_FLOW_TABLE]) +    \
2149                         offset / 32)) >>                                        \
2150           (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ)
2151 static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps)
2152 {
2153         int i;
2154
2155         for (i = 0; i < caps->arr_sz; i++) {
2156                 if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i]))
2157                         return false;
2158         }
2159         return true;
2160 }
2161
2162 static int init_root_tree_recursive(struct mlx5_flow_steering *steering,
2163                                     struct init_tree_node *init_node,
2164                                     struct fs_node *fs_parent_node,
2165                                     struct init_tree_node *init_parent_node,
2166                                     int prio)
2167 {
2168         int max_ft_level = MLX5_CAP_FLOWTABLE(steering->dev,
2169                                               flow_table_properties_nic_receive.
2170                                               max_ft_level);
2171         struct mlx5_flow_namespace *fs_ns;
2172         struct fs_prio *fs_prio;
2173         struct fs_node *base;
2174         int i;
2175         int err;
2176
2177         if (init_node->type == FS_TYPE_PRIO) {
2178                 if ((init_node->min_ft_level > max_ft_level) ||
2179                     !has_required_caps(steering->dev, &init_node->caps))
2180                         return 0;
2181
2182                 fs_get_obj(fs_ns, fs_parent_node);
2183                 if (init_node->num_leaf_prios)
2184                         return create_leaf_prios(fs_ns, prio, init_node);
2185                 fs_prio = fs_create_prio(fs_ns, prio, init_node->num_levels);
2186                 if (IS_ERR(fs_prio))
2187                         return PTR_ERR(fs_prio);
2188                 base = &fs_prio->node;
2189         } else if (init_node->type == FS_TYPE_NAMESPACE) {
2190                 fs_get_obj(fs_prio, fs_parent_node);
2191                 fs_ns = fs_create_namespace(fs_prio);
2192                 if (IS_ERR(fs_ns))
2193                         return PTR_ERR(fs_ns);
2194                 base = &fs_ns->node;
2195         } else {
2196                 return -EINVAL;
2197         }
2198         prio = 0;
2199         for (i = 0; i < init_node->ar_size; i++) {
2200                 err = init_root_tree_recursive(steering, &init_node->children[i],
2201                                                base, init_node, prio);
2202                 if (err)
2203                         return err;
2204                 if (init_node->children[i].type == FS_TYPE_PRIO &&
2205                     init_node->children[i].num_leaf_prios) {
2206                         prio += init_node->children[i].num_leaf_prios;
2207                 }
2208         }
2209
2210         return 0;
2211 }
2212
2213 static int init_root_tree(struct mlx5_flow_steering *steering,
2214                           struct init_tree_node *init_node,
2215                           struct fs_node *fs_parent_node)
2216 {
2217         int i;
2218         struct mlx5_flow_namespace *fs_ns;
2219         int err;
2220
2221         fs_get_obj(fs_ns, fs_parent_node);
2222         for (i = 0; i < init_node->ar_size; i++) {
2223                 err = init_root_tree_recursive(steering, &init_node->children[i],
2224                                                &fs_ns->node,
2225                                                init_node, i);
2226                 if (err)
2227                         return err;
2228         }
2229         return 0;
2230 }
2231
2232 static struct mlx5_flow_root_namespace *create_root_ns(struct mlx5_flow_steering *steering,
2233                                                        enum fs_flow_table_type
2234                                                        table_type)
2235 {
2236         struct mlx5_flow_root_namespace *root_ns;
2237         struct mlx5_flow_namespace *ns;
2238
2239         /* Create the root namespace */
2240         root_ns = kvzalloc(sizeof(*root_ns), GFP_KERNEL);
2241         if (!root_ns)
2242                 return NULL;
2243
2244         root_ns->dev = steering->dev;
2245         root_ns->table_type = table_type;
2246
2247         INIT_LIST_HEAD(&root_ns->underlay_qpns);
2248
2249         ns = &root_ns->ns;
2250         fs_init_namespace(ns);
2251         mutex_init(&root_ns->chain_lock);
2252         tree_init_node(&ns->node, NULL, NULL);
2253         tree_add_node(&ns->node, NULL);
2254
2255         return root_ns;
2256 }
2257
2258 static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level);
2259
2260 static int set_prio_attrs_in_ns(struct mlx5_flow_namespace *ns, int acc_level)
2261 {
2262         struct fs_prio *prio;
2263
2264         fs_for_each_prio(prio, ns) {
2265                  /* This updates prio start_level and num_levels */
2266                 set_prio_attrs_in_prio(prio, acc_level);
2267                 acc_level += prio->num_levels;
2268         }
2269         return acc_level;
2270 }
2271
2272 static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level)
2273 {
2274         struct mlx5_flow_namespace *ns;
2275         int acc_level_ns = acc_level;
2276
2277         prio->start_level = acc_level;
2278         fs_for_each_ns(ns, prio)
2279                 /* This updates start_level and num_levels of ns's priority descendants */
2280                 acc_level_ns = set_prio_attrs_in_ns(ns, acc_level);
2281         if (!prio->num_levels)
2282                 prio->num_levels = acc_level_ns - prio->start_level;
2283         WARN_ON(prio->num_levels < acc_level_ns - prio->start_level);
2284 }
2285
2286 static void set_prio_attrs(struct mlx5_flow_root_namespace *root_ns)
2287 {
2288         struct mlx5_flow_namespace *ns = &root_ns->ns;
2289         struct fs_prio *prio;
2290         int start_level = 0;
2291
2292         fs_for_each_prio(prio, ns) {
2293                 set_prio_attrs_in_prio(prio, start_level);
2294                 start_level += prio->num_levels;
2295         }
2296 }
2297
2298 #define ANCHOR_PRIO 0
2299 #define ANCHOR_SIZE 1
2300 #define ANCHOR_LEVEL 0
2301 static int create_anchor_flow_table(struct mlx5_flow_steering *steering)
2302 {
2303         struct mlx5_flow_namespace *ns = NULL;
2304         struct mlx5_flow_table_attr ft_attr = {};
2305         struct mlx5_flow_table *ft;
2306
2307         ns = mlx5_get_flow_namespace(steering->dev, MLX5_FLOW_NAMESPACE_ANCHOR);
2308         if (WARN_ON(!ns))
2309                 return -EINVAL;
2310
2311         ft_attr.max_fte = ANCHOR_SIZE;
2312         ft_attr.level   = ANCHOR_LEVEL;
2313         ft_attr.prio    = ANCHOR_PRIO;
2314
2315         ft = mlx5_create_flow_table(ns, &ft_attr);
2316         if (IS_ERR(ft)) {
2317                 mlx5_core_err(steering->dev, "Failed to create last anchor flow table");
2318                 return PTR_ERR(ft);
2319         }
2320         return 0;
2321 }
2322
2323 static int init_root_ns(struct mlx5_flow_steering *steering)
2324 {
2325         steering->root_ns = create_root_ns(steering, FS_FT_NIC_RX);
2326         if (!steering->root_ns)
2327                 goto cleanup;
2328
2329         if (init_root_tree(steering, &root_fs, &steering->root_ns->ns.node))
2330                 goto cleanup;
2331
2332         set_prio_attrs(steering->root_ns);
2333
2334         if (create_anchor_flow_table(steering))
2335                 goto cleanup;
2336
2337         return 0;
2338
2339 cleanup:
2340         mlx5_cleanup_fs(steering->dev);
2341         return -ENOMEM;
2342 }
2343
2344 static void clean_tree(struct fs_node *node)
2345 {
2346         if (node) {
2347                 struct fs_node *iter;
2348                 struct fs_node *temp;
2349
2350                 tree_get_node(node);
2351                 list_for_each_entry_safe(iter, temp, &node->children, list)
2352                         clean_tree(iter);
2353                 tree_put_node(node);
2354                 tree_remove_node(node);
2355         }
2356 }
2357
2358 static void cleanup_root_ns(struct mlx5_flow_root_namespace *root_ns)
2359 {
2360         if (!root_ns)
2361                 return;
2362
2363         clean_tree(&root_ns->ns.node);
2364 }
2365
2366 static void cleanup_egress_acls_root_ns(struct mlx5_core_dev *dev)
2367 {
2368         struct mlx5_flow_steering *steering = dev->priv.steering;
2369         int i;
2370
2371         if (!steering->esw_egress_root_ns)
2372                 return;
2373
2374         for (i = 0; i < MLX5_TOTAL_VPORTS(dev); i++)
2375                 cleanup_root_ns(steering->esw_egress_root_ns[i]);
2376
2377         kfree(steering->esw_egress_root_ns);
2378 }
2379
2380 static void cleanup_ingress_acls_root_ns(struct mlx5_core_dev *dev)
2381 {
2382         struct mlx5_flow_steering *steering = dev->priv.steering;
2383         int i;
2384
2385         if (!steering->esw_ingress_root_ns)
2386                 return;
2387
2388         for (i = 0; i < MLX5_TOTAL_VPORTS(dev); i++)
2389                 cleanup_root_ns(steering->esw_ingress_root_ns[i]);
2390
2391         kfree(steering->esw_ingress_root_ns);
2392 }
2393
2394 void mlx5_cleanup_fs(struct mlx5_core_dev *dev)
2395 {
2396         struct mlx5_flow_steering *steering = dev->priv.steering;
2397
2398         cleanup_root_ns(steering->root_ns);
2399         cleanup_egress_acls_root_ns(dev);
2400         cleanup_ingress_acls_root_ns(dev);
2401         cleanup_root_ns(steering->fdb_root_ns);
2402         cleanup_root_ns(steering->sniffer_rx_root_ns);
2403         cleanup_root_ns(steering->sniffer_tx_root_ns);
2404         mlx5_cleanup_fc_stats(dev);
2405         kmem_cache_destroy(steering->ftes_cache);
2406         kmem_cache_destroy(steering->fgs_cache);
2407         kfree(steering);
2408 }
2409
2410 static int init_sniffer_tx_root_ns(struct mlx5_flow_steering *steering)
2411 {
2412         struct fs_prio *prio;
2413
2414         steering->sniffer_tx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_TX);
2415         if (!steering->sniffer_tx_root_ns)
2416                 return -ENOMEM;
2417
2418         /* Create single prio */
2419         prio = fs_create_prio(&steering->sniffer_tx_root_ns->ns, 0, 1);
2420         if (IS_ERR(prio)) {
2421                 cleanup_root_ns(steering->sniffer_tx_root_ns);
2422                 return PTR_ERR(prio);
2423         }
2424         return 0;
2425 }
2426
2427 static int init_sniffer_rx_root_ns(struct mlx5_flow_steering *steering)
2428 {
2429         struct fs_prio *prio;
2430
2431         steering->sniffer_rx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_RX);
2432         if (!steering->sniffer_rx_root_ns)
2433                 return -ENOMEM;
2434
2435         /* Create single prio */
2436         prio = fs_create_prio(&steering->sniffer_rx_root_ns->ns, 0, 1);
2437         if (IS_ERR(prio)) {
2438                 cleanup_root_ns(steering->sniffer_rx_root_ns);
2439                 return PTR_ERR(prio);
2440         }
2441         return 0;
2442 }
2443
2444 static int init_fdb_root_ns(struct mlx5_flow_steering *steering)
2445 {
2446         struct fs_prio *prio;
2447
2448         steering->fdb_root_ns = create_root_ns(steering, FS_FT_FDB);
2449         if (!steering->fdb_root_ns)
2450                 return -ENOMEM;
2451
2452         prio = fs_create_prio(&steering->fdb_root_ns->ns, 0, 1);
2453         if (IS_ERR(prio))
2454                 goto out_err;
2455
2456         prio = fs_create_prio(&steering->fdb_root_ns->ns, 1, 1);
2457         if (IS_ERR(prio))
2458                 goto out_err;
2459
2460         set_prio_attrs(steering->fdb_root_ns);
2461         return 0;
2462
2463 out_err:
2464         cleanup_root_ns(steering->fdb_root_ns);
2465         steering->fdb_root_ns = NULL;
2466         return PTR_ERR(prio);
2467 }
2468
2469 static int init_egress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)
2470 {
2471         struct fs_prio *prio;
2472
2473         steering->esw_egress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_EGRESS_ACL);
2474         if (!steering->esw_egress_root_ns[vport])
2475                 return -ENOMEM;
2476
2477         /* create 1 prio*/
2478         prio = fs_create_prio(&steering->esw_egress_root_ns[vport]->ns, 0, 1);
2479         return PTR_ERR_OR_ZERO(prio);
2480 }
2481
2482 static int init_ingress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)
2483 {
2484         struct fs_prio *prio;
2485
2486         steering->esw_ingress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_INGRESS_ACL);
2487         if (!steering->esw_ingress_root_ns[vport])
2488                 return -ENOMEM;
2489
2490         /* create 1 prio*/
2491         prio = fs_create_prio(&steering->esw_ingress_root_ns[vport]->ns, 0, 1);
2492         return PTR_ERR_OR_ZERO(prio);
2493 }
2494
2495 static int init_egress_acls_root_ns(struct mlx5_core_dev *dev)
2496 {
2497         struct mlx5_flow_steering *steering = dev->priv.steering;
2498         int err;
2499         int i;
2500
2501         steering->esw_egress_root_ns = kcalloc(MLX5_TOTAL_VPORTS(dev),
2502                                                sizeof(*steering->esw_egress_root_ns),
2503                                                GFP_KERNEL);
2504         if (!steering->esw_egress_root_ns)
2505                 return -ENOMEM;
2506
2507         for (i = 0; i < MLX5_TOTAL_VPORTS(dev); i++) {
2508                 err = init_egress_acl_root_ns(steering, i);
2509                 if (err)
2510                         goto cleanup_root_ns;
2511         }
2512
2513         return 0;
2514
2515 cleanup_root_ns:
2516         for (i--; i >= 0; i--)
2517                 cleanup_root_ns(steering->esw_egress_root_ns[i]);
2518         kfree(steering->esw_egress_root_ns);
2519         return err;
2520 }
2521
2522 static int init_ingress_acls_root_ns(struct mlx5_core_dev *dev)
2523 {
2524         struct mlx5_flow_steering *steering = dev->priv.steering;
2525         int err;
2526         int i;
2527
2528         steering->esw_ingress_root_ns = kcalloc(MLX5_TOTAL_VPORTS(dev),
2529                                                 sizeof(*steering->esw_ingress_root_ns),
2530                                                 GFP_KERNEL);
2531         if (!steering->esw_ingress_root_ns)
2532                 return -ENOMEM;
2533
2534         for (i = 0; i < MLX5_TOTAL_VPORTS(dev); i++) {
2535                 err = init_ingress_acl_root_ns(steering, i);
2536                 if (err)
2537                         goto cleanup_root_ns;
2538         }
2539
2540         return 0;
2541
2542 cleanup_root_ns:
2543         for (i--; i >= 0; i--)
2544                 cleanup_root_ns(steering->esw_ingress_root_ns[i]);
2545         kfree(steering->esw_ingress_root_ns);
2546         return err;
2547 }
2548
2549 int mlx5_init_fs(struct mlx5_core_dev *dev)
2550 {
2551         struct mlx5_flow_steering *steering;
2552         int err = 0;
2553
2554         err = mlx5_init_fc_stats(dev);
2555         if (err)
2556                 return err;
2557
2558         steering = kzalloc(sizeof(*steering), GFP_KERNEL);
2559         if (!steering)
2560                 return -ENOMEM;
2561         steering->dev = dev;
2562         dev->priv.steering = steering;
2563
2564         steering->fgs_cache = kmem_cache_create("mlx5_fs_fgs",
2565                                                 sizeof(struct mlx5_flow_group), 0,
2566                                                 0, NULL);
2567         steering->ftes_cache = kmem_cache_create("mlx5_fs_ftes", sizeof(struct fs_fte), 0,
2568                                                  0, NULL);
2569         if (!steering->ftes_cache || !steering->fgs_cache) {
2570                 err = -ENOMEM;
2571                 goto err;
2572         }
2573
2574         if ((((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
2575               (MLX5_CAP_GEN(dev, nic_flow_table))) ||
2576              ((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_IB) &&
2577               MLX5_CAP_GEN(dev, ipoib_enhanced_offloads))) &&
2578             MLX5_CAP_FLOWTABLE_NIC_RX(dev, ft_support)) {
2579                 err = init_root_ns(steering);
2580                 if (err)
2581                         goto err;
2582         }
2583
2584         if (MLX5_CAP_GEN(dev, eswitch_flow_table)) {
2585                 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ft_support)) {
2586                         err = init_fdb_root_ns(steering);
2587                         if (err)
2588                                 goto err;
2589                 }
2590                 if (MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support)) {
2591                         err = init_egress_acls_root_ns(dev);
2592                         if (err)
2593                                 goto err;
2594                 }
2595                 if (MLX5_CAP_ESW_INGRESS_ACL(dev, ft_support)) {
2596                         err = init_ingress_acls_root_ns(dev);
2597                         if (err)
2598                                 goto err;
2599                 }
2600         }
2601
2602         if (MLX5_CAP_FLOWTABLE_SNIFFER_RX(dev, ft_support)) {
2603                 err = init_sniffer_rx_root_ns(steering);
2604                 if (err)
2605                         goto err;
2606         }
2607
2608         if (MLX5_CAP_FLOWTABLE_SNIFFER_TX(dev, ft_support)) {
2609                 err = init_sniffer_tx_root_ns(steering);
2610                 if (err)
2611                         goto err;
2612         }
2613
2614         return 0;
2615 err:
2616         mlx5_cleanup_fs(dev);
2617         return err;
2618 }
2619
2620 int mlx5_fs_add_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn)
2621 {
2622         struct mlx5_flow_root_namespace *root = dev->priv.steering->root_ns;
2623         struct mlx5_ft_underlay_qp *new_uqp;
2624         int err = 0;
2625
2626         new_uqp = kzalloc(sizeof(*new_uqp), GFP_KERNEL);
2627         if (!new_uqp)
2628                 return -ENOMEM;
2629
2630         mutex_lock(&root->chain_lock);
2631
2632         if (!root->root_ft) {
2633                 err = -EINVAL;
2634                 goto update_ft_fail;
2635         }
2636
2637         err = mlx5_cmd_update_root_ft(dev, root->root_ft, underlay_qpn, false);
2638         if (err) {
2639                 mlx5_core_warn(dev, "Failed adding underlay QPN (%u) to root FT err(%d)\n",
2640                                underlay_qpn, err);
2641                 goto update_ft_fail;
2642         }
2643
2644         new_uqp->qpn = underlay_qpn;
2645         list_add_tail(&new_uqp->list, &root->underlay_qpns);
2646
2647         mutex_unlock(&root->chain_lock);
2648
2649         return 0;
2650
2651 update_ft_fail:
2652         mutex_unlock(&root->chain_lock);
2653         kfree(new_uqp);
2654         return err;
2655 }
2656 EXPORT_SYMBOL(mlx5_fs_add_rx_underlay_qpn);
2657
2658 int mlx5_fs_remove_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn)
2659 {
2660         struct mlx5_flow_root_namespace *root = dev->priv.steering->root_ns;
2661         struct mlx5_ft_underlay_qp *uqp;
2662         bool found = false;
2663         int err = 0;
2664
2665         mutex_lock(&root->chain_lock);
2666         list_for_each_entry(uqp, &root->underlay_qpns, list) {
2667                 if (uqp->qpn == underlay_qpn) {
2668                         found = true;
2669                         break;
2670                 }
2671         }
2672
2673         if (!found) {
2674                 mlx5_core_warn(dev, "Failed finding underlay qp (%u) in qpn list\n",
2675                                underlay_qpn);
2676                 err = -EINVAL;
2677                 goto out;
2678         }
2679
2680         err = mlx5_cmd_update_root_ft(dev, root->root_ft, underlay_qpn, true);
2681         if (err)
2682                 mlx5_core_warn(dev, "Failed removing underlay QPN (%u) from root FT err(%d)\n",
2683                                underlay_qpn, err);
2684
2685         list_del(&uqp->list);
2686         mutex_unlock(&root->chain_lock);
2687         kfree(uqp);
2688
2689         return 0;
2690
2691 out:
2692         mutex_unlock(&root->chain_lock);
2693         return err;
2694 }
2695 EXPORT_SYMBOL(mlx5_fs_remove_rx_underlay_qpn);