drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
   3
   4 #include <linux/netdevice.h>
   5 #include "en_accel/fs_tcp.h"
   6 #include "fs_core.h"
   7
   8 enum accel_fs_tcp_type {
   9         ACCEL_FS_IPV4_TCP,
  10         ACCEL_FS_IPV6_TCP,
  11         ACCEL_FS_TCP_NUM_TYPES,
  12 };
  13
  14 struct mlx5e_accel_fs_tcp {
  15         struct mlx5e_flow_table tables[ACCEL_FS_TCP_NUM_TYPES];
  16         struct mlx5_flow_handle *default_rules[ACCEL_FS_TCP_NUM_TYPES];
  17 };
  18
  19 static enum mlx5e_traffic_types fs_accel2tt(enum accel_fs_tcp_type i)
  20 {
  21         switch (i) {
  22         case ACCEL_FS_IPV4_TCP:
  23                 return MLX5E_TT_IPV4_TCP;
  24         default: /* ACCEL_FS_IPV6_TCP */
  25                 return MLX5E_TT_IPV6_TCP;
  26         }
  27 }
  28
  29 static void accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec *spec, struct sock *sk)
  30 {
  31         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
  32         MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP);
  33         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
  34         MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4);
  35         memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
  36                             outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4),
  37                &inet_sk(sk)->inet_daddr, 4);
  38         memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
  39                             outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
  40                &inet_sk(sk)->inet_rcv_saddr, 4);
  41         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
  42                          outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
  43         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
  44                          outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
  45 }
  46
  47 static void accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct sock *sk)
  48 {
  49         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
  50         MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP);
  51         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
  52         MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6);
  53         memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
  54                             outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
  55                &sk->sk_v6_daddr, 16);
  56         memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
  57                             outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
  58                &inet6_sk(sk)->saddr, 16);
  59         memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
  60                             outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
  61                0xff, 16);
  62         memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
  63                             outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
  64                0xff, 16);
  65 }
  66
  67 void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule)
  68 {
  69         mlx5_del_flow_rules(rule);
  70 }
  71
  72 struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv,
  73                                                struct sock *sk, u32 tirn,
  74                                                uint32_t flow_tag)
  75 {
  76         struct mlx5_flow_destination dest = {};
  77         struct mlx5e_flow_table *ft = NULL;
  78         struct mlx5e_accel_fs_tcp *fs_tcp;
  79         MLX5_DECLARE_FLOW_ACT(flow_act);
  80         struct mlx5_flow_handle *flow;
  81         struct mlx5_flow_spec *spec;
  82
  83         spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
  84         if (!spec)
  85                 return ERR_PTR(-ENOMEM);
  86
  87         fs_tcp = priv->fs.accel_tcp;
  88
  89         spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
  90
  91         switch (sk->sk_family) {
  92         case AF_INET:
  93                 accel_fs_tcp_set_ipv4_flow(spec, sk);
  94                 ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP];
  95                 mlx5e_dbg(HW, priv, "%s flow is %pI4:%d -> %pI4:%d\n", __func__,
  96                           &inet_sk(sk)->inet_rcv_saddr,
  97                           inet_sk(sk)->inet_sport,
  98                           &inet_sk(sk)->inet_daddr,
  99                           inet_sk(sk)->inet_dport);
 100                 break;
 101 #if IS_ENABLED(CONFIG_IPV6)
 102         case AF_INET6:
 103                 if (!sk->sk_ipv6only &&
 104                     ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
 105                         accel_fs_tcp_set_ipv4_flow(spec, sk);
 106                         ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP];
 107                 } else {
 108                         accel_fs_tcp_set_ipv6_flow(spec, sk);
 109                         ft = &fs_tcp->tables[ACCEL_FS_IPV6_TCP];
 110                 }
 111                 break;
 112 #endif
 113         default:
 114                 break;
 115         }
 116
 117         if (!ft) {
 118                 flow = ERR_PTR(-EINVAL);
 119                 goto out;
 120         }
 121
 122         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
 123                          outer_headers.tcp_dport);
 124         MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
 125                          outer_headers.tcp_sport);
 126         MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport,
 127                  ntohs(inet_sk(sk)->inet_sport));
 128         MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport,
 129                  ntohs(inet_sk(sk)->inet_dport));
 130
 131         dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 132         dest.tir_num = tirn;
 133         if (flow_tag != MLX5_FS_DEFAULT_FLOW_TAG) {
 134                 spec->flow_context.flow_tag = flow_tag;
 135                 spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
 136         }
 137
 138         flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1);
 139
 140         if (IS_ERR(flow))
 141                 netdev_err(priv->netdev, "mlx5_add_flow_rules() failed, flow is %ld\n",
 142                            PTR_ERR(flow));
 143
 144 out:
 145         kvfree(spec);
 146         return flow;
 147 }
 148
 149 static int accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv,
 150                                          enum accel_fs_tcp_type type)
 151 {
 152         struct mlx5e_flow_table *accel_fs_t;
 153         struct mlx5_flow_destination dest;
 154         struct mlx5e_accel_fs_tcp *fs_tcp;
 155         MLX5_DECLARE_FLOW_ACT(flow_act);
 156         struct mlx5_flow_handle *rule;
 157         int err = 0;
 158
 159         fs_tcp = priv->fs.accel_tcp;
 160         accel_fs_t = &fs_tcp->tables[type];
 161
 162         dest = mlx5e_ttc_get_default_dest(priv, fs_accel2tt(type));
 163         rule = mlx5_add_flow_rules(accel_fs_t->t, NULL, &flow_act, &dest, 1);
 164         if (IS_ERR(rule)) {
 165                 err = PTR_ERR(rule);
 166                 netdev_err(priv->netdev,
 167                            "%s: add default rule failed, accel_fs type=%d, err %d\n",
 168                            __func__, type, err);
 169                 return err;
 170         }
 171
 172         fs_tcp->default_rules[type] = rule;
 173         return 0;
 174 }
 175
 176 #define MLX5E_ACCEL_FS_TCP_NUM_GROUPS   (2)
 177 #define MLX5E_ACCEL_FS_TCP_GROUP1_SIZE  (BIT(16) - 1)
 178 #define MLX5E_ACCEL_FS_TCP_GROUP2_SIZE  (BIT(0))
 179 #define MLX5E_ACCEL_FS_TCP_TABLE_SIZE   (MLX5E_ACCEL_FS_TCP_GROUP1_SIZE +\
 180                                          MLX5E_ACCEL_FS_TCP_GROUP2_SIZE)
 181 static int accel_fs_tcp_create_groups(struct mlx5e_flow_table *ft,
 182                                       enum accel_fs_tcp_type type)
 183 {
 184         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 185         void *outer_headers_c;
 186         int ix = 0;
 187         u32 *in;
 188         int err;
 189         u8 *mc;
 190
 191         ft->g = kcalloc(MLX5E_ACCEL_FS_TCP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
 192         in = kvzalloc(inlen, GFP_KERNEL);
 193         if  (!in || !ft->g) {
 194                 kfree(ft->g);
 195                 kvfree(in);
 196                 return -ENOMEM;
 197         }
 198
 199         mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
 200         outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
 201         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
 202         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version);
 203
 204         switch (type) {
 205         case ACCEL_FS_IPV4_TCP:
 206         case ACCEL_FS_IPV6_TCP:
 207                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport);
 208                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport);
 209                 break;
 210         default:
 211                 err = -EINVAL;
 212                 goto out;
 213         }
 214
 215         switch (type) {
 216         case ACCEL_FS_IPV4_TCP:
 217                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
 218                                  src_ipv4_src_ipv6.ipv4_layout.ipv4);
 219                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
 220                                  dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 221                 break;
 222         case ACCEL_FS_IPV6_TCP:
 223                 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
 224                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
 225                        0xff, 16);
 226                 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
 227                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 228                        0xff, 16);
 229                 break;
 230         default:
 231                 err = -EINVAL;
 232                 goto out;
 233         }
 234
 235         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
 236         MLX5_SET_CFG(in, start_flow_index, ix);
 237         ix += MLX5E_ACCEL_FS_TCP_GROUP1_SIZE;
 238         MLX5_SET_CFG(in, end_flow_index, ix - 1);
 239         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
 240         if (IS_ERR(ft->g[ft->num_groups]))
 241                 goto err;
 242         ft->num_groups++;
 243
 244         /* Default Flow Group */
 245         memset(in, 0, inlen);
 246         MLX5_SET_CFG(in, start_flow_index, ix);
 247         ix += MLX5E_ACCEL_FS_TCP_GROUP2_SIZE;
 248         MLX5_SET_CFG(in, end_flow_index, ix - 1);
 249         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
 250         if (IS_ERR(ft->g[ft->num_groups]))
 251                 goto err;
 252         ft->num_groups++;
 253
 254         kvfree(in);
 255         return 0;
 256
 257 err:
 258         err = PTR_ERR(ft->g[ft->num_groups]);
 259         ft->g[ft->num_groups] = NULL;
 260 out:
 261         kvfree(in);
 262
 263         return err;
 264 }
 265
 266 static int accel_fs_tcp_create_table(struct mlx5e_priv *priv, enum accel_fs_tcp_type type)
 267 {
 268         struct mlx5e_flow_table *ft = &priv->fs.accel_tcp->tables[type];
 269         struct mlx5_flow_table_attr ft_attr = {};
 270         int err;
 271
 272         ft->num_groups = 0;
 273
 274         ft_attr.max_fte = MLX5E_ACCEL_FS_TCP_TABLE_SIZE;
 275         ft_attr.level = MLX5E_ACCEL_FS_TCP_FT_LEVEL;
 276         ft_attr.prio = MLX5E_NIC_PRIO;
 277
 278         ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
 279         if (IS_ERR(ft->t)) {
 280                 err = PTR_ERR(ft->t);
 281                 ft->t = NULL;
 282                 return err;
 283         }
 284
 285         netdev_dbg(priv->netdev, "Created fs accel table id %u level %u\n",
 286                    ft->t->id, ft->t->level);
 287
 288         err = accel_fs_tcp_create_groups(ft, type);
 289         if (err)
 290                 goto err;
 291
 292         err = accel_fs_tcp_add_default_rule(priv, type);
 293         if (err)
 294                 goto err;
 295
 296         return 0;
 297 err:
 298         mlx5e_destroy_flow_table(ft);
 299         return err;
 300 }
 301
 302 static int accel_fs_tcp_disable(struct mlx5e_priv *priv)
 303 {
 304         int err, i;
 305
 306         for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) {
 307                 /* Modify ttc rules destination to point back to the indir TIRs */
 308                 err = mlx5e_ttc_fwd_default_dest(priv, fs_accel2tt(i));
 309                 if (err) {
 310                         netdev_err(priv->netdev,
 311                                    "%s: modify ttc[%d] default destination failed, err(%d)\n",
 312                                    __func__, fs_accel2tt(i), err);
 313                         return err;
 314                 }
 315         }
 316
 317         return 0;
 318 }
 319
 320 static int accel_fs_tcp_enable(struct mlx5e_priv *priv)
 321 {
 322         struct mlx5_flow_destination dest = {};
 323         int err, i;
 324
 325         dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 326         for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) {
 327                 dest.ft = priv->fs.accel_tcp->tables[i].t;
 328
 329                 /* Modify ttc rules destination to point on the accel_fs FTs */
 330                 err = mlx5e_ttc_fwd_dest(priv, fs_accel2tt(i), &dest);
 331                 if (err) {
 332                         netdev_err(priv->netdev,
 333                                    "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
 334                                    __func__, fs_accel2tt(i), err);
 335                         return err;
 336                 }
 337         }
 338         return 0;
 339 }
 340
 341 static void accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i)
 342 {
 343         struct mlx5e_accel_fs_tcp *fs_tcp;
 344
 345         fs_tcp = priv->fs.accel_tcp;
 346         if (IS_ERR_OR_NULL(fs_tcp->tables[i].t))
 347                 return;
 348
 349         mlx5_del_flow_rules(fs_tcp->default_rules[i]);
 350         mlx5e_destroy_flow_table(&fs_tcp->tables[i]);
 351         fs_tcp->tables[i].t = NULL;
 352 }
 353
 354 void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv)
 355 {
 356         int i;
 357
 358         if (!priv->fs.accel_tcp)
 359                 return;
 360
 361         accel_fs_tcp_disable(priv);
 362
 363         for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++)
 364                 accel_fs_tcp_destroy_table(priv, i);
 365
 366         kfree(priv->fs.accel_tcp);
 367         priv->fs.accel_tcp = NULL;
 368 }
 369
 370 int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv)
 371 {
 372         int i, err;
 373
 374         if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version))
 375                 return -EOPNOTSUPP;
 376
 377         priv->fs.accel_tcp = kzalloc(sizeof(*priv->fs.accel_tcp), GFP_KERNEL);
 378         if (!priv->fs.accel_tcp)
 379                 return -ENOMEM;
 380
 381         for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) {
 382                 err = accel_fs_tcp_create_table(priv, i);
 383                 if (err)
 384                         goto err_destroy_tables;
 385         }
 386
 387         err = accel_fs_tcp_enable(priv);
 388         if (err)
 389                 goto err_destroy_tables;
 390
 391         return 0;
 392
 393 err_destroy_tables:
 394         while (--i >= 0)
 395                 accel_fs_tcp_destroy_table(priv, i);
 396
 397         kfree(priv->fs.accel_tcp);
 398         priv->fs.accel_tcp = NULL;
 399         return err;
 400 }