drivers/net/ethernet/mellanox/mlx5/core/en_tc.c

   1 /*
   2  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <net/flow_dissector.h>
  34 #include <net/sch_generic.h>
  35 #include <net/pkt_cls.h>
  36 #include <net/tc_act/tc_gact.h>
  37 #include <net/tc_act/tc_skbedit.h>
  38 #include <linux/mlx5/fs.h>
  39 #include <linux/mlx5/device.h>
  40 #include <linux/rhashtable.h>
  41 #include <net/switchdev.h>
  42 #include <net/tc_act/tc_mirred.h>
  43 #include <net/tc_act/tc_vlan.h>
  44 #include <net/tc_act/tc_tunnel_key.h>
  45 #include <net/vxlan.h>
  46 #include "en.h"
  47 #include "en_tc.h"
  48 #include "eswitch.h"
  49 #include "vxlan.h"
  50
  51 struct mlx5e_tc_flow {
  52         struct rhash_head       node;
  53         u64                     cookie;
  54         struct mlx5_flow_handle *rule;
  55         struct list_head        encap; /* flows sharing the same encap */
  56         struct mlx5_esw_flow_attr *attr;
  57 };
  58
  59 enum {
  60         MLX5_HEADER_TYPE_VXLAN = 0x0,
  61         MLX5_HEADER_TYPE_NVGRE = 0x1,
  62 };
  63
  64 #define MLX5E_TC_TABLE_NUM_ENTRIES 1024
  65 #define MLX5E_TC_TABLE_NUM_GROUPS 4
  66
  67 static struct mlx5_flow_handle *
  68 mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
  69                       struct mlx5_flow_spec *spec,
  70                       u32 action, u32 flow_tag)
  71 {
  72         struct mlx5_core_dev *dev = priv->mdev;
  73         struct mlx5_flow_destination dest = { 0 };
  74         struct mlx5_flow_act flow_act = {
  75                 .action = action,
  76                 .flow_tag = flow_tag,
  77                 .encap_id = 0,
  78         };
  79         struct mlx5_fc *counter = NULL;
  80         struct mlx5_flow_handle *rule;
  81         bool table_created = false;
  82
  83         if (action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
  84                 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
  85                 dest.ft = priv->fs.vlan.ft.t;
  86         } else if (action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
  87                 counter = mlx5_fc_create(dev, true);
  88                 if (IS_ERR(counter))
  89                         return ERR_CAST(counter);
  90
  91                 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
  92                 dest.counter = counter;
  93         }
  94
  95         if (IS_ERR_OR_NULL(priv->fs.tc.t)) {
  96                 priv->fs.tc.t =
  97                         mlx5_create_auto_grouped_flow_table(priv->fs.ns,
  98                                                             MLX5E_TC_PRIO,
  99                                                             MLX5E_TC_TABLE_NUM_ENTRIES,
 100                                                             MLX5E_TC_TABLE_NUM_GROUPS,
 101                                                             0, 0);
 102                 if (IS_ERR(priv->fs.tc.t)) {
 103                         netdev_err(priv->netdev,
 104                                    "Failed to create tc offload table\n");
 105                         rule = ERR_CAST(priv->fs.tc.t);
 106                         goto err_create_ft;
 107                 }
 108
 109                 table_created = true;
 110         }
 111
 112         spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 113         rule = mlx5_add_flow_rules(priv->fs.tc.t, spec, &flow_act, &dest, 1);
 114
 115         if (IS_ERR(rule))
 116                 goto err_add_rule;
 117
 118         return rule;
 119
 120 err_add_rule:
 121         if (table_created) {
 122                 mlx5_destroy_flow_table(priv->fs.tc.t);
 123                 priv->fs.tc.t = NULL;
 124         }
 125 err_create_ft:
 126         mlx5_fc_destroy(dev, counter);
 127
 128         return rule;
 129 }
 130
 131 static struct mlx5_flow_handle *
 132 mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv,
 133                       struct mlx5_flow_spec *spec,
 134                       struct mlx5_esw_flow_attr *attr)
 135 {
 136         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 137         int err;
 138
 139         err = mlx5_eswitch_add_vlan_action(esw, attr);
 140         if (err)
 141                 return ERR_PTR(err);
 142
 143         return mlx5_eswitch_add_offloaded_rule(esw, spec, attr);
 144 }
 145
 146 static void mlx5e_detach_encap(struct mlx5e_priv *priv,
 147                                struct mlx5e_tc_flow *flow) {
 148         struct list_head *next = flow->encap.next;
 149
 150         list_del(&flow->encap);
 151         if (list_empty(next)) {
 152                 struct mlx5_encap_entry *e;
 153
 154                 e = list_entry(next, struct mlx5_encap_entry, flows);
 155                 if (e->n) {
 156                         mlx5_encap_dealloc(priv->mdev, e->encap_id);
 157                         neigh_release(e->n);
 158                 }
 159                 hlist_del_rcu(&e->encap_hlist);
 160                 kfree(e);
 161         }
 162 }
 163
 164 static void mlx5e_tc_del_flow(struct mlx5e_priv *priv,
 165                               struct mlx5e_tc_flow *flow)
 166 {
 167         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 168         struct mlx5_fc *counter = NULL;
 169
 170         counter = mlx5_flow_rule_counter(flow->rule);
 171
 172         mlx5_del_flow_rules(flow->rule);
 173
 174         if (esw && esw->mode == SRIOV_OFFLOADS) {
 175                 mlx5_eswitch_del_vlan_action(esw, flow->attr);
 176                 if (flow->attr->action & MLX5_FLOW_CONTEXT_ACTION_ENCAP)
 177                         mlx5e_detach_encap(priv, flow);
 178         }
 179
 180         mlx5_fc_destroy(priv->mdev, counter);
 181
 182         if (!mlx5e_tc_num_filters(priv) && (priv->fs.tc.t)) {
 183                 mlx5_destroy_flow_table(priv->fs.tc.t);
 184                 priv->fs.tc.t = NULL;
 185         }
 186 }
 187
 188 static void parse_vxlan_attr(struct mlx5_flow_spec *spec,
 189                              struct tc_cls_flower_offload *f)
 190 {
 191         void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 192                                        outer_headers);
 193         void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 194                                        outer_headers);
 195         void *misc_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 196                                     misc_parameters);
 197         void *misc_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 198                                     misc_parameters);
 199
 200         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol);
 201         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
 202
 203         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
 204                 struct flow_dissector_key_keyid *key =
 205                         skb_flow_dissector_target(f->dissector,
 206                                                   FLOW_DISSECTOR_KEY_ENC_KEYID,
 207                                                   f->key);
 208                 struct flow_dissector_key_keyid *mask =
 209                         skb_flow_dissector_target(f->dissector,
 210                                                   FLOW_DISSECTOR_KEY_ENC_KEYID,
 211                                                   f->mask);
 212                 MLX5_SET(fte_match_set_misc, misc_c, vxlan_vni,
 213                          be32_to_cpu(mask->keyid));
 214                 MLX5_SET(fte_match_set_misc, misc_v, vxlan_vni,
 215                          be32_to_cpu(key->keyid));
 216         }
 217 }
 218
 219 static int parse_tunnel_attr(struct mlx5e_priv *priv,
 220                              struct mlx5_flow_spec *spec,
 221                              struct tc_cls_flower_offload *f)
 222 {
 223         void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 224                                        outer_headers);
 225         void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 226                                        outer_headers);
 227
 228         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
 229                 struct flow_dissector_key_ports *key =
 230                         skb_flow_dissector_target(f->dissector,
 231                                                   FLOW_DISSECTOR_KEY_ENC_PORTS,
 232                                                   f->key);
 233                 struct flow_dissector_key_ports *mask =
 234                         skb_flow_dissector_target(f->dissector,
 235                                                   FLOW_DISSECTOR_KEY_ENC_PORTS,
 236                                                   f->mask);
 237
 238                 /* Full udp dst port must be given */
 239                 if (memchr_inv(&mask->dst, 0xff, sizeof(mask->dst)))
 240                         return -EOPNOTSUPP;
 241
 242                 /* udp src port isn't supported */
 243                 if (memchr_inv(&mask->src, 0, sizeof(mask->src)))
 244                         return -EOPNOTSUPP;
 245
 246                 if (mlx5e_vxlan_lookup_port(priv, be16_to_cpu(key->dst)) &&
 247                     MLX5_CAP_ESW(priv->mdev, vxlan_encap_decap))
 248                         parse_vxlan_attr(spec, f);
 249                 else
 250                         return -EOPNOTSUPP;
 251
 252                 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 253                          udp_dport, ntohs(mask->dst));
 254                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 255                          udp_dport, ntohs(key->dst));
 256
 257         } else { /* udp dst port must be given */
 258                         return -EOPNOTSUPP;
 259         }
 260
 261         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
 262                 struct flow_dissector_key_ipv4_addrs *key =
 263                         skb_flow_dissector_target(f->dissector,
 264                                                   FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
 265                                                   f->key);
 266                 struct flow_dissector_key_ipv4_addrs *mask =
 267                         skb_flow_dissector_target(f->dissector,
 268                                                   FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
 269                                                   f->mask);
 270                 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 271                          src_ipv4_src_ipv6.ipv4_layout.ipv4,
 272                          ntohl(mask->src));
 273                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 274                          src_ipv4_src_ipv6.ipv4_layout.ipv4,
 275                          ntohl(key->src));
 276
 277                 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 278                          dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
 279                          ntohl(mask->dst));
 280                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 281                          dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
 282                          ntohl(key->dst));
 283         }
 284
 285         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ethertype);
 286         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, ETH_P_IP);
 287
 288         /* Enforce DMAC when offloading incoming tunneled flows.
 289          * Flow counters require a match on the DMAC.
 290          */
 291         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, dmac_47_16);
 292         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, dmac_15_0);
 293         ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 294                                      dmac_47_16), priv->netdev->dev_addr);
 295
 296         /* let software handle IP fragments */
 297         MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
 298         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
 299
 300         return 0;
 301 }
 302
 303 static int __parse_cls_flower(struct mlx5e_priv *priv,
 304                               struct mlx5_flow_spec *spec,
 305                               struct tc_cls_flower_offload *f,
 306                               u8 *min_inline)
 307 {
 308         void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 309                                        outer_headers);
 310         void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 311                                        outer_headers);
 312         u16 addr_type = 0;
 313         u8 ip_proto = 0;
 314
 315         *min_inline = MLX5_INLINE_MODE_L2;
 316
 317         if (f->dissector->used_keys &
 318             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
 319               BIT(FLOW_DISSECTOR_KEY_BASIC) |
 320               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
 321               BIT(FLOW_DISSECTOR_KEY_VLAN) |
 322               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
 323               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
 324               BIT(FLOW_DISSECTOR_KEY_PORTS) |
 325               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) |
 326               BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
 327               BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
 328               BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) |
 329               BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL))) {
 330                 netdev_warn(priv->netdev, "Unsupported key used: 0x%x\n",
 331                             f->dissector->used_keys);
 332                 return -EOPNOTSUPP;
 333         }
 334
 335         if ((dissector_uses_key(f->dissector,
 336                                 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) ||
 337              dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID) ||
 338              dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) &&
 339             dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
 340                 struct flow_dissector_key_control *key =
 341                         skb_flow_dissector_target(f->dissector,
 342                                                   FLOW_DISSECTOR_KEY_ENC_CONTROL,
 343                                                   f->key);
 344                 switch (key->addr_type) {
 345                 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
 346                         if (parse_tunnel_attr(priv, spec, f))
 347                                 return -EOPNOTSUPP;
 348                         break;
 349                 default:
 350                         return -EOPNOTSUPP;
 351                 }
 352
 353                 /* In decap flow, header pointers should point to the inner
 354                  * headers, outer header were already set by parse_tunnel_attr
 355                  */
 356                 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 357                                          inner_headers);
 358                 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 359                                          inner_headers);
 360         }
 361
 362         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
 363                 struct flow_dissector_key_control *key =
 364                         skb_flow_dissector_target(f->dissector,
 365                                                   FLOW_DISSECTOR_KEY_CONTROL,
 366                                                   f->key);
 367
 368                 struct flow_dissector_key_control *mask =
 369                         skb_flow_dissector_target(f->dissector,
 370                                                   FLOW_DISSECTOR_KEY_CONTROL,
 371                                                   f->mask);
 372                 addr_type = key->addr_type;
 373
 374                 if (mask->flags & FLOW_DIS_IS_FRAGMENT) {
 375                         MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
 376                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
 377                                  key->flags & FLOW_DIS_IS_FRAGMENT);
 378                 }
 379         }
 380
 381         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
 382                 struct flow_dissector_key_basic *key =
 383                         skb_flow_dissector_target(f->dissector,
 384                                                   FLOW_DISSECTOR_KEY_BASIC,
 385                                                   f->key);
 386                 struct flow_dissector_key_basic *mask =
 387                         skb_flow_dissector_target(f->dissector,
 388                                                   FLOW_DISSECTOR_KEY_BASIC,
 389                                                   f->mask);
 390                 ip_proto = key->ip_proto;
 391
 392                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype,
 393                          ntohs(mask->n_proto));
 394                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
 395                          ntohs(key->n_proto));
 396
 397                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
 398                          mask->ip_proto);
 399                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 400                          key->ip_proto);
 401
 402                 if (mask->ip_proto)
 403                         *min_inline = MLX5_INLINE_MODE_IP;
 404         }
 405
 406         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
 407                 struct flow_dissector_key_eth_addrs *key =
 408                         skb_flow_dissector_target(f->dissector,
 409                                                   FLOW_DISSECTOR_KEY_ETH_ADDRS,
 410                                                   f->key);
 411                 struct flow_dissector_key_eth_addrs *mask =
 412                         skb_flow_dissector_target(f->dissector,
 413                                                   FLOW_DISSECTOR_KEY_ETH_ADDRS,
 414                                                   f->mask);
 415
 416                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 417                                              dmac_47_16),
 418                                 mask->dst);
 419                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 420                                              dmac_47_16),
 421                                 key->dst);
 422
 423                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 424                                              smac_47_16),
 425                                 mask->src);
 426                 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 427                                              smac_47_16),
 428                                 key->src);
 429         }
 430
 431         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
 432                 struct flow_dissector_key_vlan *key =
 433                         skb_flow_dissector_target(f->dissector,
 434                                                   FLOW_DISSECTOR_KEY_VLAN,
 435                                                   f->key);
 436                 struct flow_dissector_key_vlan *mask =
 437                         skb_flow_dissector_target(f->dissector,
 438                                                   FLOW_DISSECTOR_KEY_VLAN,
 439                                                   f->mask);
 440                 if (mask->vlan_id || mask->vlan_priority) {
 441                         MLX5_SET(fte_match_set_lyr_2_4, headers_c, vlan_tag, 1);
 442                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, vlan_tag, 1);
 443
 444                         MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_vid, mask->vlan_id);
 445                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, key->vlan_id);
 446
 447                         MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_prio, mask->vlan_priority);
 448                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, key->vlan_priority);
 449                 }
 450         }
 451
 452         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
 453                 struct flow_dissector_key_ipv4_addrs *key =
 454                         skb_flow_dissector_target(f->dissector,
 455                                                   FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 456                                                   f->key);
 457                 struct flow_dissector_key_ipv4_addrs *mask =
 458                         skb_flow_dissector_target(f->dissector,
 459                                                   FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 460                                                   f->mask);
 461
 462                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 463                                     src_ipv4_src_ipv6.ipv4_layout.ipv4),
 464                        &mask->src, sizeof(mask->src));
 465                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 466                                     src_ipv4_src_ipv6.ipv4_layout.ipv4),
 467                        &key->src, sizeof(key->src));
 468                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 469                                     dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 470                        &mask->dst, sizeof(mask->dst));
 471                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 472                                     dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 473                        &key->dst, sizeof(key->dst));
 474
 475                 if (mask->src || mask->dst)
 476                         *min_inline = MLX5_INLINE_MODE_IP;
 477         }
 478
 479         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
 480                 struct flow_dissector_key_ipv6_addrs *key =
 481                         skb_flow_dissector_target(f->dissector,
 482                                                   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
 483                                                   f->key);
 484                 struct flow_dissector_key_ipv6_addrs *mask =
 485                         skb_flow_dissector_target(f->dissector,
 486                                                   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
 487                                                   f->mask);
 488
 489                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 490                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
 491                        &mask->src, sizeof(mask->src));
 492                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 493                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
 494                        &key->src, sizeof(key->src));
 495
 496                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 497                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 498                        &mask->dst, sizeof(mask->dst));
 499                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 500                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 501                        &key->dst, sizeof(key->dst));
 502
 503                 if (ipv6_addr_type(&mask->src) != IPV6_ADDR_ANY ||
 504                     ipv6_addr_type(&mask->dst) != IPV6_ADDR_ANY)
 505                         *min_inline = MLX5_INLINE_MODE_IP;
 506         }
 507
 508         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) {
 509                 struct flow_dissector_key_ports *key =
 510                         skb_flow_dissector_target(f->dissector,
 511                                                   FLOW_DISSECTOR_KEY_PORTS,
 512                                                   f->key);
 513                 struct flow_dissector_key_ports *mask =
 514                         skb_flow_dissector_target(f->dissector,
 515                                                   FLOW_DISSECTOR_KEY_PORTS,
 516                                                   f->mask);
 517                 switch (ip_proto) {
 518                 case IPPROTO_TCP:
 519                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 520                                  tcp_sport, ntohs(mask->src));
 521                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 522                                  tcp_sport, ntohs(key->src));
 523
 524                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 525                                  tcp_dport, ntohs(mask->dst));
 526                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 527                                  tcp_dport, ntohs(key->dst));
 528                         break;
 529
 530                 case IPPROTO_UDP:
 531                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 532                                  udp_sport, ntohs(mask->src));
 533                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 534                                  udp_sport, ntohs(key->src));
 535
 536                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 537                                  udp_dport, ntohs(mask->dst));
 538                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 539                                  udp_dport, ntohs(key->dst));
 540                         break;
 541                 default:
 542                         netdev_err(priv->netdev,
 543                                    "Only UDP and TCP transport are supported\n");
 544                         return -EINVAL;
 545                 }
 546
 547                 if (mask->src || mask->dst)
 548                         *min_inline = MLX5_INLINE_MODE_TCP_UDP;
 549         }
 550
 551         return 0;
 552 }
 553
 554 static int parse_cls_flower(struct mlx5e_priv *priv,
 555                             struct mlx5_flow_spec *spec,
 556                             struct tc_cls_flower_offload *f)
 557 {
 558         struct mlx5_core_dev *dev = priv->mdev;
 559         struct mlx5_eswitch *esw = dev->priv.eswitch;
 560         struct mlx5_eswitch_rep *rep = priv->ppriv;
 561         u8 min_inline;
 562         int err;
 563
 564         err = __parse_cls_flower(priv, spec, f, &min_inline);
 565
 566         if (!err && esw->mode == SRIOV_OFFLOADS &&
 567             rep->vport != FDB_UPLINK_VPORT) {
 568                 if (min_inline > esw->offloads.inline_mode) {
 569                         netdev_warn(priv->netdev,
 570                                     "Flow is not offloaded due to min inline setting, required %d actual %d\n",
 571                                     min_inline, esw->offloads.inline_mode);
 572                         return -EOPNOTSUPP;
 573                 }
 574         }
 575
 576         return err;
 577 }
 578
 579 static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 580                                 u32 *action, u32 *flow_tag)
 581 {
 582         const struct tc_action *a;
 583         LIST_HEAD(actions);
 584
 585         if (tc_no_actions(exts))
 586                 return -EINVAL;
 587
 588         *flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
 589         *action = 0;
 590
 591         tcf_exts_to_list(exts, &actions);
 592         list_for_each_entry(a, &actions, list) {
 593                 /* Only support a single action per rule */
 594                 if (*action)
 595                         return -EINVAL;
 596
 597                 if (is_tcf_gact_shot(a)) {
 598                         *action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
 599                         if (MLX5_CAP_FLOWTABLE(priv->mdev,
 600                                                flow_table_properties_nic_receive.flow_counter))
 601                                 *action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
 602                         continue;
 603                 }
 604
 605                 if (is_tcf_skbedit_mark(a)) {
 606                         u32 mark = tcf_skbedit_mark(a);
 607
 608                         if (mark & ~MLX5E_TC_FLOW_ID_MASK) {
 609                                 netdev_warn(priv->netdev, "Bad flow mark - only 16 bit is supported: 0x%x\n",
 610                                             mark);
 611                                 return -EINVAL;
 612                         }
 613
 614                         *flow_tag = mark;
 615                         *action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 616                         continue;
 617                 }
 618
 619                 return -EINVAL;
 620         }
 621
 622         return 0;
 623 }
 624
 625 static inline int cmp_encap_info(struct mlx5_encap_info *a,
 626                                  struct mlx5_encap_info *b)
 627 {
 628         return memcmp(a, b, sizeof(*a));
 629 }
 630
 631 static inline int hash_encap_info(struct mlx5_encap_info *info)
 632 {
 633         return jhash(info, sizeof(*info), 0);
 634 }
 635
 636 static int mlx5e_route_lookup_ipv4(struct mlx5e_priv *priv,
 637                                    struct net_device *mirred_dev,
 638                                    struct net_device **out_dev,
 639                                    struct flowi4 *fl4,
 640                                    struct neighbour **out_n,
 641                                    __be32 *saddr,
 642                                    int *out_ttl)
 643 {
 644         struct rtable *rt;
 645         struct neighbour *n = NULL;
 646         int ttl;
 647
 648 #if IS_ENABLED(CONFIG_INET)
 649         rt = ip_route_output_key(dev_net(mirred_dev), fl4);
 650         if (IS_ERR(rt)) {
 651                 pr_warn("%s: no route to %pI4\n", __func__, &fl4->daddr);
 652                 return -EOPNOTSUPP;
 653         }
 654 #else
 655         return -EOPNOTSUPP;
 656 #endif
 657
 658         if (!switchdev_port_same_parent_id(priv->netdev, rt->dst.dev)) {
 659                 pr_warn("%s: Can't offload the flow, netdevices aren't on the same HW e-switch\n",
 660                         __func__);
 661                 ip_rt_put(rt);
 662                 return -EOPNOTSUPP;
 663         }
 664
 665         ttl = ip4_dst_hoplimit(&rt->dst);
 666         n = dst_neigh_lookup(&rt->dst, &fl4->daddr);
 667         ip_rt_put(rt);
 668         if (!n)
 669                 return -ENOMEM;
 670
 671         *out_n = n;
 672         *saddr = fl4->saddr;
 673         *out_ttl = ttl;
 674         *out_dev = rt->dst.dev;
 675
 676         return 0;
 677 }
 678
 679 static int gen_vxlan_header_ipv4(struct net_device *out_dev,
 680                                  char buf[],
 681                                  unsigned char h_dest[ETH_ALEN],
 682                                  int ttl,
 683                                  __be32 daddr,
 684                                  __be32 saddr,
 685                                  __be16 udp_dst_port,
 686                                  __be32 vx_vni)
 687 {
 688         int encap_size = VXLAN_HLEN + sizeof(struct iphdr) + ETH_HLEN;
 689         struct ethhdr *eth = (struct ethhdr *)buf;
 690         struct iphdr  *ip = (struct iphdr *)((char *)eth + sizeof(struct ethhdr));
 691         struct udphdr *udp = (struct udphdr *)((char *)ip + sizeof(struct iphdr));
 692         struct vxlanhdr *vxh = (struct vxlanhdr *)((char *)udp + sizeof(struct udphdr));
 693
 694         memset(buf, 0, encap_size);
 695
 696         ether_addr_copy(eth->h_dest, h_dest);
 697         ether_addr_copy(eth->h_source, out_dev->dev_addr);
 698         eth->h_proto = htons(ETH_P_IP);
 699
 700         ip->daddr = daddr;
 701         ip->saddr = saddr;
 702
 703         ip->ttl = ttl;
 704         ip->protocol = IPPROTO_UDP;
 705         ip->version = 0x4;
 706         ip->ihl = 0x5;
 707
 708         udp->dest = udp_dst_port;
 709         vxh->vx_flags = VXLAN_HF_VNI;
 710         vxh->vx_vni = vxlan_vni_field(vx_vni);
 711
 712         return encap_size;
 713 }
 714
 715 static int mlx5e_create_encap_header_ipv4(struct mlx5e_priv *priv,
 716                                           struct net_device *mirred_dev,
 717                                           struct mlx5_encap_entry *e,
 718                                           struct net_device **out_dev)
 719 {
 720         int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
 721         struct flowi4 fl4 = {};
 722         struct neighbour *n;
 723         char *encap_header;
 724         int encap_size;
 725         __be32 saddr;
 726         int ttl;
 727         int err;
 728
 729         encap_header = kzalloc(max_encap_size, GFP_KERNEL);
 730         if (!encap_header)
 731                 return -ENOMEM;
 732
 733         switch (e->tunnel_type) {
 734         case MLX5_HEADER_TYPE_VXLAN:
 735                 fl4.flowi4_proto = IPPROTO_UDP;
 736                 fl4.fl4_dport = e->tun_info.tp_dst;
 737                 break;
 738         default:
 739                 err = -EOPNOTSUPP;
 740                 goto out;
 741         }
 742         fl4.daddr = e->tun_info.daddr;
 743
 744         err = mlx5e_route_lookup_ipv4(priv, mirred_dev, out_dev,
 745                                       &fl4, &n, &saddr, &ttl);
 746         if (err)
 747                 goto out;
 748
 749         e->n = n;
 750         e->out_dev = *out_dev;
 751
 752         if (!(n->nud_state & NUD_VALID)) {
 753                 err = -ENOTSUPP;
 754                 goto out;
 755         }
 756
 757         neigh_ha_snapshot(e->h_dest, n, *out_dev);
 758
 759         switch (e->tunnel_type) {
 760         case MLX5_HEADER_TYPE_VXLAN:
 761                 encap_size = gen_vxlan_header_ipv4(*out_dev, encap_header,
 762                                                    e->h_dest, ttl,
 763                                                    e->tun_info.daddr,
 764                                                    saddr, e->tun_info.tp_dst,
 765                                                    e->tun_info.tun_id);
 766                 break;
 767         default:
 768                 err = -EOPNOTSUPP;
 769                 goto out;
 770         }
 771
 772         err = mlx5_encap_alloc(priv->mdev, e->tunnel_type,
 773                                encap_size, encap_header, &e->encap_id);
 774 out:
 775         kfree(encap_header);
 776         return err;
 777 }
 778
 779 static int mlx5e_attach_encap(struct mlx5e_priv *priv,
 780                               struct ip_tunnel_info *tun_info,
 781                               struct net_device *mirred_dev,
 782                               struct mlx5_esw_flow_attr *attr)
 783 {
 784         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 785         unsigned short family = ip_tunnel_info_af(tun_info);
 786         struct ip_tunnel_key *key = &tun_info->key;
 787         struct mlx5_encap_info info;
 788         struct mlx5_encap_entry *e;
 789         struct net_device *out_dev;
 790         uintptr_t hash_key;
 791         bool found = false;
 792         int tunnel_type;
 793         int err;
 794
 795         /* udp dst port must be given */
 796         if (!memchr_inv(&key->tp_dst, 0, sizeof(key->tp_dst)))
 797                 return -EOPNOTSUPP;
 798
 799         if (mlx5e_vxlan_lookup_port(priv, be16_to_cpu(key->tp_dst)) &&
 800             MLX5_CAP_ESW(priv->mdev, vxlan_encap_decap)) {
 801                 info.tp_dst = key->tp_dst;
 802                 info.tun_id = tunnel_id_to_key32(key->tun_id);
 803                 tunnel_type = MLX5_HEADER_TYPE_VXLAN;
 804         } else {
 805                 return -EOPNOTSUPP;
 806         }
 807
 808         switch (family) {
 809         case AF_INET:
 810                 info.daddr = key->u.ipv4.dst;
 811                 break;
 812         default:
 813                 return -EOPNOTSUPP;
 814         }
 815
 816         hash_key = hash_encap_info(&info);
 817
 818         hash_for_each_possible_rcu(esw->offloads.encap_tbl, e,
 819                                    encap_hlist, hash_key) {
 820                 if (!cmp_encap_info(&e->tun_info, &info)) {
 821                         found = true;
 822                         break;
 823                 }
 824         }
 825
 826         if (found) {
 827                 attr->encap = e;
 828                 return 0;
 829         }
 830
 831         e = kzalloc(sizeof(*e), GFP_KERNEL);
 832         if (!e)
 833                 return -ENOMEM;
 834
 835         e->tun_info = info;
 836         e->tunnel_type = tunnel_type;
 837         INIT_LIST_HEAD(&e->flows);
 838
 839         err = mlx5e_create_encap_header_ipv4(priv, mirred_dev, e, &out_dev);
 840         if (err)
 841                 goto out_err;
 842
 843         attr->encap = e;
 844         hash_add_rcu(esw->offloads.encap_tbl, &e->encap_hlist, hash_key);
 845
 846         return err;
 847
 848 out_err:
 849         kfree(e);
 850         return err;
 851 }
 852
 853 static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 854                                 struct mlx5e_tc_flow *flow)
 855 {
 856         struct mlx5_esw_flow_attr *attr = flow->attr;
 857         struct ip_tunnel_info *info = NULL;
 858         const struct tc_action *a;
 859         LIST_HEAD(actions);
 860         bool encap = false;
 861         int err;
 862
 863         if (tc_no_actions(exts))
 864                 return -EINVAL;
 865
 866         memset(attr, 0, sizeof(*attr));
 867         attr->in_rep = priv->ppriv;
 868
 869         tcf_exts_to_list(exts, &actions);
 870         list_for_each_entry(a, &actions, list) {
 871                 if (is_tcf_gact_shot(a)) {
 872                         attr->action |= MLX5_FLOW_CONTEXT_ACTION_DROP |
 873                                         MLX5_FLOW_CONTEXT_ACTION_COUNT;
 874                         continue;
 875                 }
 876
 877                 if (is_tcf_mirred_egress_redirect(a)) {
 878                         int ifindex = tcf_mirred_ifindex(a);
 879                         struct net_device *out_dev;
 880                         struct mlx5e_priv *out_priv;
 881
 882                         out_dev = __dev_get_by_index(dev_net(priv->netdev), ifindex);
 883
 884                         if (switchdev_port_same_parent_id(priv->netdev,
 885                                                           out_dev)) {
 886                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
 887                                         MLX5_FLOW_CONTEXT_ACTION_COUNT;
 888                                 out_priv = netdev_priv(out_dev);
 889                                 attr->out_rep = out_priv->ppriv;
 890                         } else if (encap) {
 891                                 err = mlx5e_attach_encap(priv, info,
 892                                                          out_dev, attr);
 893                                 if (err)
 894                                         return err;
 895                                 list_add(&flow->encap, &attr->encap->flows);
 896                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_ENCAP |
 897                                         MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
 898                                         MLX5_FLOW_CONTEXT_ACTION_COUNT;
 899                                 out_priv = netdev_priv(attr->encap->out_dev);
 900                                 attr->out_rep = out_priv->ppriv;
 901                         } else {
 902                                 pr_err("devices %s %s not on same switch HW, can't offload forwarding\n",
 903                                        priv->netdev->name, out_dev->name);
 904                                 return -EINVAL;
 905                         }
 906                         continue;
 907                 }
 908
 909                 if (is_tcf_tunnel_set(a)) {
 910                         info = tcf_tunnel_info(a);
 911                         if (info)
 912                                 encap = true;
 913                         else
 914                                 return -EOPNOTSUPP;
 915                         continue;
 916                 }
 917
 918                 if (is_tcf_vlan(a)) {
 919                         if (tcf_vlan_action(a) == VLAN_F_POP) {
 920                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP;
 921                         } else if (tcf_vlan_action(a) == VLAN_F_PUSH) {
 922                                 if (tcf_vlan_push_proto(a) != htons(ETH_P_8021Q))
 923                                         return -EOPNOTSUPP;
 924
 925                                 attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
 926                                 attr->vlan = tcf_vlan_push_vid(a);
 927                         }
 928                         continue;
 929                 }
 930
 931                 if (is_tcf_tunnel_release(a)) {
 932                         attr->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
 933                         continue;
 934                 }
 935
 936                 return -EINVAL;
 937         }
 938         return 0;
 939 }
 940
 941 int mlx5e_configure_flower(struct mlx5e_priv *priv, __be16 protocol,
 942                            struct tc_cls_flower_offload *f)
 943 {
 944         struct mlx5e_tc_table *tc = &priv->fs.tc;
 945         int err = 0;
 946         bool fdb_flow = false;
 947         u32 flow_tag, action;
 948         struct mlx5e_tc_flow *flow;
 949         struct mlx5_flow_spec *spec;
 950         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 951
 952         if (esw && esw->mode == SRIOV_OFFLOADS)
 953                 fdb_flow = true;
 954
 955         if (fdb_flow)
 956                 flow = kzalloc(sizeof(*flow) +
 957                                sizeof(struct mlx5_esw_flow_attr),
 958                                GFP_KERNEL);
 959         else
 960                 flow = kzalloc(sizeof(*flow), GFP_KERNEL);
 961
 962         spec = mlx5_vzalloc(sizeof(*spec));
 963         if (!spec || !flow) {
 964                 err = -ENOMEM;
 965                 goto err_free;
 966         }
 967
 968         flow->cookie = f->cookie;
 969
 970         err = parse_cls_flower(priv, spec, f);
 971         if (err < 0)
 972                 goto err_free;
 973
 974         if (fdb_flow) {
 975                 flow->attr  = (struct mlx5_esw_flow_attr *)(flow + 1);
 976                 err = parse_tc_fdb_actions(priv, f->exts, flow);
 977                 if (err < 0)
 978                         goto err_free;
 979                 flow->rule = mlx5e_tc_add_fdb_flow(priv, spec, flow->attr);
 980         } else {
 981                 err = parse_tc_nic_actions(priv, f->exts, &action, &flow_tag);
 982                 if (err < 0)
 983                         goto err_free;
 984                 flow->rule = mlx5e_tc_add_nic_flow(priv, spec, action, flow_tag);
 985         }
 986
 987         if (IS_ERR(flow->rule)) {
 988                 err = PTR_ERR(flow->rule);
 989                 goto err_free;
 990         }
 991
 992         err = rhashtable_insert_fast(&tc->ht, &flow->node,
 993                                      tc->ht_params);
 994         if (err)
 995                 goto err_del_rule;
 996
 997         goto out;
 998
 999 err_del_rule:
1000         mlx5_del_flow_rules(flow->rule);
1001
1002 err_free:
1003         kfree(flow);
1004 out:
1005         kvfree(spec);
1006         return err;
1007 }
1008
1009 int mlx5e_delete_flower(struct mlx5e_priv *priv,
1010                         struct tc_cls_flower_offload *f)
1011 {
1012         struct mlx5e_tc_flow *flow;
1013         struct mlx5e_tc_table *tc = &priv->fs.tc;
1014
1015         flow = rhashtable_lookup_fast(&tc->ht, &f->cookie,
1016                                       tc->ht_params);
1017         if (!flow)
1018                 return -EINVAL;
1019
1020         rhashtable_remove_fast(&tc->ht, &flow->node, tc->ht_params);
1021
1022         mlx5e_tc_del_flow(priv, flow);
1023
1024
1025         kfree(flow);
1026
1027         return 0;
1028 }
1029
1030 int mlx5e_stats_flower(struct mlx5e_priv *priv,
1031                        struct tc_cls_flower_offload *f)
1032 {
1033         struct mlx5e_tc_table *tc = &priv->fs.tc;
1034         struct mlx5e_tc_flow *flow;
1035         struct tc_action *a;
1036         struct mlx5_fc *counter;
1037         LIST_HEAD(actions);
1038         u64 bytes;
1039         u64 packets;
1040         u64 lastuse;
1041
1042         flow = rhashtable_lookup_fast(&tc->ht, &f->cookie,
1043                                       tc->ht_params);
1044         if (!flow)
1045                 return -EINVAL;
1046
1047         counter = mlx5_flow_rule_counter(flow->rule);
1048         if (!counter)
1049                 return 0;
1050
1051         mlx5_fc_query_cached(counter, &bytes, &packets, &lastuse);
1052
1053         tcf_exts_to_list(f->exts, &actions);
1054         list_for_each_entry(a, &actions, list)
1055                 tcf_action_stats_update(a, bytes, packets, lastuse);
1056
1057         return 0;
1058 }
1059
1060 static const struct rhashtable_params mlx5e_tc_flow_ht_params = {
1061         .head_offset = offsetof(struct mlx5e_tc_flow, node),
1062         .key_offset = offsetof(struct mlx5e_tc_flow, cookie),
1063         .key_len = sizeof(((struct mlx5e_tc_flow *)0)->cookie),
1064         .automatic_shrinking = true,
1065 };
1066
1067 int mlx5e_tc_init(struct mlx5e_priv *priv)
1068 {
1069         struct mlx5e_tc_table *tc = &priv->fs.tc;
1070
1071         tc->ht_params = mlx5e_tc_flow_ht_params;
1072         return rhashtable_init(&tc->ht, &tc->ht_params);
1073 }
1074
1075 static void _mlx5e_tc_del_flow(void *ptr, void *arg)
1076 {
1077         struct mlx5e_tc_flow *flow = ptr;
1078         struct mlx5e_priv *priv = arg;
1079
1080         mlx5e_tc_del_flow(priv, flow);
1081         kfree(flow);
1082 }
1083
1084 void mlx5e_tc_cleanup(struct mlx5e_priv *priv)
1085 {
1086         struct mlx5e_tc_table *tc = &priv->fs.tc;
1087
1088         rhashtable_free_and_destroy(&tc->ht, _mlx5e_tc_del_flow, priv);
1089
1090         if (!IS_ERR_OR_NULL(tc->t)) {
1091                 mlx5_destroy_flow_table(tc->t);
1092                 tc->t = NULL;
1093         }
1094 }