drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2017 Broadcom Limited
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  */
   9
  10 #include <linux/netdevice.h>
  11 #include <linux/inetdevice.h>
  12 #include <linux/if_vlan.h>
  13 #include <net/flow_dissector.h>
  14 #include <net/pkt_cls.h>
  15 #include <net/tc_act/tc_gact.h>
  16 #include <net/tc_act/tc_skbedit.h>
  17 #include <net/tc_act/tc_mirred.h>
  18 #include <net/tc_act/tc_vlan.h>
  19
  20 #include "bnxt_hsi.h"
  21 #include "bnxt.h"
  22 #include "bnxt_sriov.h"
  23 #include "bnxt_tc.h"
  24 #include "bnxt_vfr.h"
  25
  26 #ifdef CONFIG_BNXT_FLOWER_OFFLOAD
  27
  28 #define BNXT_FID_INVALID                        0xffff
  29 #define VLAN_TCI(vid, prio)     ((vid) | ((prio) << VLAN_PRIO_SHIFT))
  30
  31 /* Return the dst fid of the func for flow forwarding
  32  * For PFs: src_fid is the fid of the PF
  33  * For VF-reps: src_fid the fid of the VF
  34  */
  35 static u16 bnxt_flow_get_dst_fid(struct bnxt *pf_bp, struct net_device *dev)
  36 {
  37         struct bnxt *bp;
  38
  39         /* check if dev belongs to the same switch */
  40         if (!switchdev_port_same_parent_id(pf_bp->dev, dev)) {
  41                 netdev_info(pf_bp->dev, "dev(ifindex=%d) not on same switch",
  42                             dev->ifindex);
  43                 return BNXT_FID_INVALID;
  44         }
  45
  46         /* Is dev a VF-rep? */
  47         if (dev != pf_bp->dev)
  48                 return bnxt_vf_rep_get_fid(dev);
  49
  50         bp = netdev_priv(dev);
  51         return bp->pf.fw_fid;
  52 }
  53
  54 static int bnxt_tc_parse_redir(struct bnxt *bp,
  55                                struct bnxt_tc_actions *actions,
  56                                const struct tc_action *tc_act)
  57 {
  58         int ifindex = tcf_mirred_ifindex(tc_act);
  59         struct net_device *dev;
  60         u16 dst_fid;
  61
  62         dev = __dev_get_by_index(dev_net(bp->dev), ifindex);
  63         if (!dev) {
  64                 netdev_info(bp->dev, "no dev for ifindex=%d", ifindex);
  65                 return -EINVAL;
  66         }
  67
  68         /* find the FID from dev */
  69         dst_fid = bnxt_flow_get_dst_fid(bp, dev);
  70         if (dst_fid == BNXT_FID_INVALID) {
  71                 netdev_info(bp->dev, "can't get fid for ifindex=%d", ifindex);
  72                 return -EINVAL;
  73         }
  74
  75         actions->flags |= BNXT_TC_ACTION_FLAG_FWD;
  76         actions->dst_fid = dst_fid;
  77         actions->dst_dev = dev;
  78         return 0;
  79 }
  80
  81 static void bnxt_tc_parse_vlan(struct bnxt *bp,
  82                                struct bnxt_tc_actions *actions,
  83                                const struct tc_action *tc_act)
  84 {
  85         if (tcf_vlan_action(tc_act) == TCA_VLAN_ACT_POP) {
  86                 actions->flags |= BNXT_TC_ACTION_FLAG_POP_VLAN;
  87         } else if (tcf_vlan_action(tc_act) == TCA_VLAN_ACT_PUSH) {
  88                 actions->flags |= BNXT_TC_ACTION_FLAG_PUSH_VLAN;
  89                 actions->push_vlan_tci = htons(tcf_vlan_push_vid(tc_act));
  90                 actions->push_vlan_tpid = tcf_vlan_push_proto(tc_act);
  91         }
  92 }
  93
  94 static int bnxt_tc_parse_actions(struct bnxt *bp,
  95                                  struct bnxt_tc_actions *actions,
  96                                  struct tcf_exts *tc_exts)
  97 {
  98         const struct tc_action *tc_act;
  99         LIST_HEAD(tc_actions);
 100         int rc;
 101
 102         if (!tcf_exts_has_actions(tc_exts)) {
 103                 netdev_info(bp->dev, "no actions");
 104                 return -EINVAL;
 105         }
 106
 107         tcf_exts_to_list(tc_exts, &tc_actions);
 108         list_for_each_entry(tc_act, &tc_actions, list) {
 109                 /* Drop action */
 110                 if (is_tcf_gact_shot(tc_act)) {
 111                         actions->flags |= BNXT_TC_ACTION_FLAG_DROP;
 112                         return 0; /* don't bother with other actions */
 113                 }
 114
 115                 /* Redirect action */
 116                 if (is_tcf_mirred_egress_redirect(tc_act)) {
 117                         rc = bnxt_tc_parse_redir(bp, actions, tc_act);
 118                         if (rc)
 119                                 return rc;
 120                         continue;
 121                 }
 122
 123                 /* Push/pop VLAN */
 124                 if (is_tcf_vlan(tc_act)) {
 125                         bnxt_tc_parse_vlan(bp, actions, tc_act);
 126                         continue;
 127                 }
 128         }
 129
 130         return 0;
 131 }
 132
 133 #define GET_KEY(flow_cmd, key_type)                                     \
 134                 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\
 135                                           (flow_cmd)->key)
 136 #define GET_MASK(flow_cmd, key_type)                                    \
 137                 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\
 138                                           (flow_cmd)->mask)
 139
 140 static int bnxt_tc_parse_flow(struct bnxt *bp,
 141                               struct tc_cls_flower_offload *tc_flow_cmd,
 142                               struct bnxt_tc_flow *flow)
 143 {
 144         struct flow_dissector *dissector = tc_flow_cmd->dissector;
 145         u16 addr_type = 0;
 146
 147         /* KEY_CONTROL and KEY_BASIC are needed for forming a meaningful key */
 148         if ((dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) == 0 ||
 149             (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) == 0) {
 150                 netdev_info(bp->dev, "cannot form TC key: used_keys = 0x%x",
 151                             dissector->used_keys);
 152                 return -EOPNOTSUPP;
 153         }
 154
 155         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
 156                 struct flow_dissector_key_control *key =
 157                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_CONTROL);
 158
 159                 addr_type = key->addr_type;
 160         }
 161
 162         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_BASIC)) {
 163                 struct flow_dissector_key_basic *key =
 164                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC);
 165                 struct flow_dissector_key_basic *mask =
 166                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC);
 167
 168                 flow->l2_key.ether_type = key->n_proto;
 169                 flow->l2_mask.ether_type = mask->n_proto;
 170
 171                 if (key->n_proto == htons(ETH_P_IP) ||
 172                     key->n_proto == htons(ETH_P_IPV6)) {
 173                         flow->l4_key.ip_proto = key->ip_proto;
 174                         flow->l4_mask.ip_proto = mask->ip_proto;
 175                 }
 176         }
 177
 178         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
 179                 struct flow_dissector_key_eth_addrs *key =
 180                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS);
 181                 struct flow_dissector_key_eth_addrs *mask =
 182                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS);
 183
 184                 flow->flags |= BNXT_TC_FLOW_FLAGS_ETH_ADDRS;
 185                 ether_addr_copy(flow->l2_key.dmac, key->dst);
 186                 ether_addr_copy(flow->l2_mask.dmac, mask->dst);
 187                 ether_addr_copy(flow->l2_key.smac, key->src);
 188                 ether_addr_copy(flow->l2_mask.smac, mask->src);
 189         }
 190
 191         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_VLAN)) {
 192                 struct flow_dissector_key_vlan *key =
 193                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN);
 194                 struct flow_dissector_key_vlan *mask =
 195                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN);
 196
 197                 flow->l2_key.inner_vlan_tci =
 198                    cpu_to_be16(VLAN_TCI(key->vlan_id, key->vlan_priority));
 199                 flow->l2_mask.inner_vlan_tci =
 200                    cpu_to_be16((VLAN_TCI(mask->vlan_id, mask->vlan_priority)));
 201                 flow->l2_key.inner_vlan_tpid = htons(ETH_P_8021Q);
 202                 flow->l2_mask.inner_vlan_tpid = htons(0xffff);
 203                 flow->l2_key.num_vlans = 1;
 204         }
 205
 206         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
 207                 struct flow_dissector_key_ipv4_addrs *key =
 208                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
 209                 struct flow_dissector_key_ipv4_addrs *mask =
 210                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
 211
 212                 flow->flags |= BNXT_TC_FLOW_FLAGS_IPV4_ADDRS;
 213                 flow->l3_key.ipv4.daddr.s_addr = key->dst;
 214                 flow->l3_mask.ipv4.daddr.s_addr = mask->dst;
 215                 flow->l3_key.ipv4.saddr.s_addr = key->src;
 216                 flow->l3_mask.ipv4.saddr.s_addr = mask->src;
 217         } else if (dissector_uses_key(dissector,
 218                                       FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
 219                 struct flow_dissector_key_ipv6_addrs *key =
 220                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
 221                 struct flow_dissector_key_ipv6_addrs *mask =
 222                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
 223
 224                 flow->flags |= BNXT_TC_FLOW_FLAGS_IPV6_ADDRS;
 225                 flow->l3_key.ipv6.daddr = key->dst;
 226                 flow->l3_mask.ipv6.daddr = mask->dst;
 227                 flow->l3_key.ipv6.saddr = key->src;
 228                 flow->l3_mask.ipv6.saddr = mask->src;
 229         }
 230
 231         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_PORTS)) {
 232                 struct flow_dissector_key_ports *key =
 233                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS);
 234                 struct flow_dissector_key_ports *mask =
 235                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS);
 236
 237                 flow->flags |= BNXT_TC_FLOW_FLAGS_PORTS;
 238                 flow->l4_key.ports.dport = key->dst;
 239                 flow->l4_mask.ports.dport = mask->dst;
 240                 flow->l4_key.ports.sport = key->src;
 241                 flow->l4_mask.ports.sport = mask->src;
 242         }
 243
 244         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ICMP)) {
 245                 struct flow_dissector_key_icmp *key =
 246                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP);
 247                 struct flow_dissector_key_icmp *mask =
 248                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP);
 249
 250                 flow->flags |= BNXT_TC_FLOW_FLAGS_ICMP;
 251                 flow->l4_key.icmp.type = key->type;
 252                 flow->l4_key.icmp.code = key->code;
 253                 flow->l4_mask.icmp.type = mask->type;
 254                 flow->l4_mask.icmp.code = mask->code;
 255         }
 256
 257         return bnxt_tc_parse_actions(bp, &flow->actions, tc_flow_cmd->exts);
 258 }
 259
 260 static int bnxt_hwrm_cfa_flow_free(struct bnxt *bp, __le16 flow_handle)
 261 {
 262         struct hwrm_cfa_flow_free_input req = { 0 };
 263         int rc;
 264
 265         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_FREE, -1, -1);
 266         req.flow_handle = flow_handle;
 267
 268         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 269         if (rc)
 270                 netdev_info(bp->dev, "Error: %s: flow_handle=0x%x rc=%d",
 271                             __func__, flow_handle, rc);
 272         return rc;
 273 }
 274
 275 static int ipv6_mask_len(struct in6_addr *mask)
 276 {
 277         int mask_len = 0, i;
 278
 279         for (i = 0; i < 4; i++)
 280                 mask_len += inet_mask_len(mask->s6_addr32[i]);
 281
 282         return mask_len;
 283 }
 284
 285 static bool is_wildcard(void *mask, int len)
 286 {
 287         const u8 *p = mask;
 288         int i;
 289
 290         for (i = 0; i < len; i++) {
 291                 if (p[i] != 0)
 292                         return false;
 293         }
 294         return true;
 295 }
 296
 297 static int bnxt_hwrm_cfa_flow_alloc(struct bnxt *bp, struct bnxt_tc_flow *flow,
 298                                     __le16 ref_flow_handle, __le16 *flow_handle)
 299 {
 300         struct hwrm_cfa_flow_alloc_output *resp = bp->hwrm_cmd_resp_addr;
 301         struct bnxt_tc_actions *actions = &flow->actions;
 302         struct bnxt_tc_l3_key *l3_mask = &flow->l3_mask;
 303         struct bnxt_tc_l3_key *l3_key = &flow->l3_key;
 304         struct hwrm_cfa_flow_alloc_input req = { 0 };
 305         u16 flow_flags = 0, action_flags = 0;
 306         int rc;
 307
 308         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_ALLOC, -1, -1);
 309
 310         req.src_fid = cpu_to_le16(flow->src_fid);
 311         req.ref_flow_handle = ref_flow_handle;
 312         req.ethertype = flow->l2_key.ether_type;
 313         req.ip_proto = flow->l4_key.ip_proto;
 314
 315         if (flow->flags & BNXT_TC_FLOW_FLAGS_ETH_ADDRS) {
 316                 memcpy(req.dmac, flow->l2_key.dmac, ETH_ALEN);
 317                 memcpy(req.smac, flow->l2_key.smac, ETH_ALEN);
 318         }
 319
 320         if (flow->l2_key.num_vlans > 0) {
 321                 flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE;
 322                 /* FW expects the inner_vlan_tci value to be set
 323                  * in outer_vlan_tci when num_vlans is 1 (which is
 324                  * always the case in TC.)
 325                  */
 326                 req.outer_vlan_tci = flow->l2_key.inner_vlan_tci;
 327         }
 328
 329         /* If all IP and L4 fields are wildcarded then this is an L2 flow */
 330         if (is_wildcard(&l3_mask, sizeof(l3_mask)) &&
 331             is_wildcard(&flow->l4_mask, sizeof(flow->l4_mask))) {
 332                 flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2;
 333         } else {
 334                 flow_flags |= flow->l2_key.ether_type == htons(ETH_P_IP) ?
 335                                 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4 :
 336                                 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6;
 337
 338                 if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV4_ADDRS) {
 339                         req.ip_dst[0] = l3_key->ipv4.daddr.s_addr;
 340                         req.ip_dst_mask_len =
 341                                 inet_mask_len(l3_mask->ipv4.daddr.s_addr);
 342                         req.ip_src[0] = l3_key->ipv4.saddr.s_addr;
 343                         req.ip_src_mask_len =
 344                                 inet_mask_len(l3_mask->ipv4.saddr.s_addr);
 345                 } else if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV6_ADDRS) {
 346                         memcpy(req.ip_dst, l3_key->ipv6.daddr.s6_addr32,
 347                                sizeof(req.ip_dst));
 348                         req.ip_dst_mask_len =
 349                                         ipv6_mask_len(&l3_mask->ipv6.daddr);
 350                         memcpy(req.ip_src, l3_key->ipv6.saddr.s6_addr32,
 351                                sizeof(req.ip_src));
 352                         req.ip_src_mask_len =
 353                                         ipv6_mask_len(&l3_mask->ipv6.saddr);
 354                 }
 355         }
 356
 357         if (flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) {
 358                 req.l4_src_port = flow->l4_key.ports.sport;
 359                 req.l4_src_port_mask = flow->l4_mask.ports.sport;
 360                 req.l4_dst_port = flow->l4_key.ports.dport;
 361                 req.l4_dst_port_mask = flow->l4_mask.ports.dport;
 362         } else if (flow->flags & BNXT_TC_FLOW_FLAGS_ICMP) {
 363                 /* l4 ports serve as type/code when ip_proto is ICMP */
 364                 req.l4_src_port = htons(flow->l4_key.icmp.type);
 365                 req.l4_src_port_mask = htons(flow->l4_mask.icmp.type);
 366                 req.l4_dst_port = htons(flow->l4_key.icmp.code);
 367                 req.l4_dst_port_mask = htons(flow->l4_mask.icmp.code);
 368         }
 369         req.flags = cpu_to_le16(flow_flags);
 370
 371         if (actions->flags & BNXT_TC_ACTION_FLAG_DROP) {
 372                 action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP;
 373         } else {
 374                 if (actions->flags & BNXT_TC_ACTION_FLAG_FWD) {
 375                         action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD;
 376                         req.dst_fid = cpu_to_le16(actions->dst_fid);
 377                 }
 378                 if (actions->flags & BNXT_TC_ACTION_FLAG_PUSH_VLAN) {
 379                         action_flags |=
 380                             CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE;
 381                         req.l2_rewrite_vlan_tpid = actions->push_vlan_tpid;
 382                         req.l2_rewrite_vlan_tci = actions->push_vlan_tci;
 383                         memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN);
 384                         memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN);
 385                 }
 386                 if (actions->flags & BNXT_TC_ACTION_FLAG_POP_VLAN) {
 387                         action_flags |=
 388                             CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE;
 389                         /* Rewrite config with tpid = 0 implies vlan pop */
 390                         req.l2_rewrite_vlan_tpid = 0;
 391                         memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN);
 392                         memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN);
 393                 }
 394         }
 395         req.action_flags = cpu_to_le16(action_flags);
 396
 397         mutex_lock(&bp->hwrm_cmd_lock);
 398
 399         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 400         if (!rc)
 401                 *flow_handle = resp->flow_handle;
 402
 403         mutex_unlock(&bp->hwrm_cmd_lock);
 404
 405         return rc;
 406 }
 407
 408 /* Add val to accum while handling a possible wraparound
 409  * of val. Eventhough val is of type u64, its actual width
 410  * is denoted by mask and will wrap-around beyond that width.
 411  */
 412 static void accumulate_val(u64 *accum, u64 val, u64 mask)
 413 {
 414 #define low_bits(x, mask)               ((x) & (mask))
 415 #define high_bits(x, mask)              ((x) & ~(mask))
 416         bool wrapped = val < low_bits(*accum, mask);
 417
 418         *accum = high_bits(*accum, mask) + val;
 419         if (wrapped)
 420                 *accum += (mask + 1);
 421 }
 422
 423 /* The HW counters' width is much less than 64bits.
 424  * Handle possible wrap-around while updating the stat counters
 425  */
 426 static void bnxt_flow_stats_fix_wraparound(struct bnxt_tc_info *tc_info,
 427                                            struct bnxt_tc_flow_stats *stats,
 428                                            struct bnxt_tc_flow_stats *hw_stats)
 429 {
 430         accumulate_val(&stats->bytes, hw_stats->bytes, tc_info->bytes_mask);
 431         accumulate_val(&stats->packets, hw_stats->packets,
 432                        tc_info->packets_mask);
 433 }
 434
 435 /* Fix possible wraparound of the stats queried from HW, calculate
 436  * the delta from prev_stats, and also update the prev_stats.
 437  * The HW flow stats are fetched under the hwrm_cmd_lock mutex.
 438  * This routine is best called while under the mutex so that the
 439  * stats processing happens atomically.
 440  */
 441 static void bnxt_flow_stats_calc(struct bnxt_tc_info *tc_info,
 442                                  struct bnxt_tc_flow *flow,
 443                                  struct bnxt_tc_flow_stats *stats)
 444 {
 445         struct bnxt_tc_flow_stats *acc_stats, *prev_stats;
 446
 447         acc_stats = &flow->stats;
 448         bnxt_flow_stats_fix_wraparound(tc_info, acc_stats, stats);
 449
 450         prev_stats = &flow->prev_stats;
 451         stats->bytes = acc_stats->bytes - prev_stats->bytes;
 452         stats->packets = acc_stats->packets - prev_stats->packets;
 453         *prev_stats = *acc_stats;
 454 }
 455
 456 static int bnxt_hwrm_cfa_flow_stats_get(struct bnxt *bp,
 457                                         __le16 flow_handle,
 458                                         struct bnxt_tc_flow *flow,
 459                                         struct bnxt_tc_flow_stats *stats)
 460 {
 461         struct hwrm_cfa_flow_stats_output *resp = bp->hwrm_cmd_resp_addr;
 462         struct hwrm_cfa_flow_stats_input req = { 0 };
 463         int rc;
 464
 465         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_STATS, -1, -1);
 466         req.num_flows = cpu_to_le16(1);
 467         req.flow_handle_0 = flow_handle;
 468
 469         mutex_lock(&bp->hwrm_cmd_lock);
 470         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 471         if (!rc) {
 472                 stats->packets = le64_to_cpu(resp->packet_0);
 473                 stats->bytes = le64_to_cpu(resp->byte_0);
 474                 bnxt_flow_stats_calc(&bp->tc_info, flow, stats);
 475         } else {
 476                 netdev_info(bp->dev, "error rc=%d", rc);
 477         }
 478
 479         mutex_unlock(&bp->hwrm_cmd_lock);
 480         return rc;
 481 }
 482
 483 static int bnxt_tc_put_l2_node(struct bnxt *bp,
 484                                struct bnxt_tc_flow_node *flow_node)
 485 {
 486         struct bnxt_tc_l2_node *l2_node = flow_node->l2_node;
 487         struct bnxt_tc_info *tc_info = &bp->tc_info;
 488         int rc;
 489
 490         /* remove flow_node from the L2 shared flow list */
 491         list_del(&flow_node->l2_list_node);
 492         if (--l2_node->refcount == 0) {
 493                 rc =  rhashtable_remove_fast(&tc_info->l2_table, &l2_node->node,
 494                                              tc_info->l2_ht_params);
 495                 if (rc)
 496                         netdev_err(bp->dev,
 497                                    "Error: %s: rhashtable_remove_fast: %d",
 498                                    __func__, rc);
 499                 kfree_rcu(l2_node, rcu);
 500         }
 501         return 0;
 502 }
 503
 504 static struct bnxt_tc_l2_node *
 505 bnxt_tc_get_l2_node(struct bnxt *bp, struct rhashtable *l2_table,
 506                     struct rhashtable_params ht_params,
 507                     struct bnxt_tc_l2_key *l2_key)
 508 {
 509         struct bnxt_tc_l2_node *l2_node;
 510         int rc;
 511
 512         l2_node = rhashtable_lookup_fast(l2_table, l2_key, ht_params);
 513         if (!l2_node) {
 514                 l2_node = kzalloc(sizeof(*l2_node), GFP_KERNEL);
 515                 if (!l2_node) {
 516                         rc = -ENOMEM;
 517                         return NULL;
 518                 }
 519
 520                 l2_node->key = *l2_key;
 521                 rc = rhashtable_insert_fast(l2_table, &l2_node->node,
 522                                             ht_params);
 523                 if (rc) {
 524                         kfree(l2_node);
 525                         netdev_err(bp->dev,
 526                                    "Error: %s: rhashtable_insert_fast: %d",
 527                                    __func__, rc);
 528                         return NULL;
 529                 }
 530                 INIT_LIST_HEAD(&l2_node->common_l2_flows);
 531         }
 532         return l2_node;
 533 }
 534
 535 /* Get the ref_flow_handle for a flow by checking if there are any other
 536  * flows that share the same L2 key as this flow.
 537  */
 538 static int
 539 bnxt_tc_get_ref_flow_handle(struct bnxt *bp, struct bnxt_tc_flow *flow,
 540                             struct bnxt_tc_flow_node *flow_node,
 541                             __le16 *ref_flow_handle)
 542 {
 543         struct bnxt_tc_info *tc_info = &bp->tc_info;
 544         struct bnxt_tc_flow_node *ref_flow_node;
 545         struct bnxt_tc_l2_node *l2_node;
 546
 547         l2_node = bnxt_tc_get_l2_node(bp, &tc_info->l2_table,
 548                                       tc_info->l2_ht_params,
 549                                       &flow->l2_key);
 550         if (!l2_node)
 551                 return -1;
 552
 553         /* If any other flow is using this l2_node, use it's flow_handle
 554          * as the ref_flow_handle
 555          */
 556         if (l2_node->refcount > 0) {
 557                 ref_flow_node = list_first_entry(&l2_node->common_l2_flows,
 558                                                  struct bnxt_tc_flow_node,
 559                                                  l2_list_node);
 560                 *ref_flow_handle = ref_flow_node->flow_handle;
 561         } else {
 562                 *ref_flow_handle = cpu_to_le16(0xffff);
 563         }
 564
 565         /* Insert the l2_node into the flow_node so that subsequent flows
 566          * with a matching l2 key can use the flow_handle of this flow
 567          * as their ref_flow_handle
 568          */
 569         flow_node->l2_node = l2_node;
 570         list_add(&flow_node->l2_list_node, &l2_node->common_l2_flows);
 571         l2_node->refcount++;
 572         return 0;
 573 }
 574
 575 /* After the flow parsing is done, this routine is used for checking
 576  * if there are any aspects of the flow that prevent it from being
 577  * offloaded.
 578  */
 579 static bool bnxt_tc_can_offload(struct bnxt *bp, struct bnxt_tc_flow *flow)
 580 {
 581         /* If L4 ports are specified then ip_proto must be TCP or UDP */
 582         if ((flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) &&
 583             (flow->l4_key.ip_proto != IPPROTO_TCP &&
 584              flow->l4_key.ip_proto != IPPROTO_UDP)) {
 585                 netdev_info(bp->dev, "Cannot offload non-TCP/UDP (%d) ports",
 586                             flow->l4_key.ip_proto);
 587                 return false;
 588         }
 589
 590         return true;
 591 }
 592
 593 static int __bnxt_tc_del_flow(struct bnxt *bp,
 594                               struct bnxt_tc_flow_node *flow_node)
 595 {
 596         struct bnxt_tc_info *tc_info = &bp->tc_info;
 597         int rc;
 598
 599         /* send HWRM cmd to free the flow-id */
 600         bnxt_hwrm_cfa_flow_free(bp, flow_node->flow_handle);
 601
 602         mutex_lock(&tc_info->lock);
 603
 604         /* release reference to l2 node */
 605         bnxt_tc_put_l2_node(bp, flow_node);
 606
 607         mutex_unlock(&tc_info->lock);
 608
 609         rc = rhashtable_remove_fast(&tc_info->flow_table, &flow_node->node,
 610                                     tc_info->flow_ht_params);
 611         if (rc)
 612                 netdev_err(bp->dev, "Error: %s: rhashtable_remove_fast rc=%d",
 613                            __func__, rc);
 614
 615         kfree_rcu(flow_node, rcu);
 616         return 0;
 617 }
 618
 619 /* Add a new flow or replace an existing flow.
 620  * Notes on locking:
 621  * There are essentially two critical sections here.
 622  * 1. while adding a new flow
 623  *    a) lookup l2-key
 624  *    b) issue HWRM cmd and get flow_handle
 625  *    c) link l2-key with flow
 626  * 2. while deleting a flow
 627  *    a) unlinking l2-key from flow
 628  * A lock is needed to protect these two critical sections.
 629  *
 630  * The hash-tables are already protected by the rhashtable API.
 631  */
 632 static int bnxt_tc_add_flow(struct bnxt *bp, u16 src_fid,
 633                             struct tc_cls_flower_offload *tc_flow_cmd)
 634 {
 635         struct bnxt_tc_flow_node *new_node, *old_node;
 636         struct bnxt_tc_info *tc_info = &bp->tc_info;
 637         struct bnxt_tc_flow *flow;
 638         __le16 ref_flow_handle;
 639         int rc;
 640
 641         /* allocate memory for the new flow and it's node */
 642         new_node = kzalloc(sizeof(*new_node), GFP_KERNEL);
 643         if (!new_node) {
 644                 rc = -ENOMEM;
 645                 goto done;
 646         }
 647         new_node->cookie = tc_flow_cmd->cookie;
 648         flow = &new_node->flow;
 649
 650         rc = bnxt_tc_parse_flow(bp, tc_flow_cmd, flow);
 651         if (rc)
 652                 goto free_node;
 653         flow->src_fid = src_fid;
 654
 655         if (!bnxt_tc_can_offload(bp, flow)) {
 656                 rc = -ENOSPC;
 657                 goto free_node;
 658         }
 659
 660         /* If a flow exists with the same cookie, delete it */
 661         old_node = rhashtable_lookup_fast(&tc_info->flow_table,
 662                                           &tc_flow_cmd->cookie,
 663                                           tc_info->flow_ht_params);
 664         if (old_node)
 665                 __bnxt_tc_del_flow(bp, old_node);
 666
 667         /* Check if the L2 part of the flow has been offloaded already.
 668          * If so, bump up it's refcnt and get it's reference handle.
 669          */
 670         mutex_lock(&tc_info->lock);
 671         rc = bnxt_tc_get_ref_flow_handle(bp, flow, new_node, &ref_flow_handle);
 672         if (rc)
 673                 goto unlock;
 674
 675         /* send HWRM cmd to alloc the flow */
 676         rc = bnxt_hwrm_cfa_flow_alloc(bp, flow, ref_flow_handle,
 677                                       &new_node->flow_handle);
 678         if (rc)
 679                 goto put_l2;
 680
 681         /* add new flow to flow-table */
 682         rc = rhashtable_insert_fast(&tc_info->flow_table, &new_node->node,
 683                                     tc_info->flow_ht_params);
 684         if (rc)
 685                 goto hwrm_flow_free;
 686
 687         mutex_unlock(&tc_info->lock);
 688         return 0;
 689
 690 hwrm_flow_free:
 691         bnxt_hwrm_cfa_flow_free(bp, new_node->flow_handle);
 692 put_l2:
 693         bnxt_tc_put_l2_node(bp, new_node);
 694 unlock:
 695         mutex_unlock(&tc_info->lock);
 696 free_node:
 697         kfree(new_node);
 698 done:
 699         netdev_err(bp->dev, "Error: %s: cookie=0x%lx error=%d",
 700                    __func__, tc_flow_cmd->cookie, rc);
 701         return rc;
 702 }
 703
 704 static int bnxt_tc_del_flow(struct bnxt *bp,
 705                             struct tc_cls_flower_offload *tc_flow_cmd)
 706 {
 707         struct bnxt_tc_info *tc_info = &bp->tc_info;
 708         struct bnxt_tc_flow_node *flow_node;
 709
 710         flow_node = rhashtable_lookup_fast(&tc_info->flow_table,
 711                                            &tc_flow_cmd->cookie,
 712                                            tc_info->flow_ht_params);
 713         if (!flow_node) {
 714                 netdev_info(bp->dev, "ERROR: no flow_node for cookie %lx",
 715                             tc_flow_cmd->cookie);
 716                 return -EINVAL;
 717         }
 718
 719         return __bnxt_tc_del_flow(bp, flow_node);
 720 }
 721
 722 static int bnxt_tc_get_flow_stats(struct bnxt *bp,
 723                                   struct tc_cls_flower_offload *tc_flow_cmd)
 724 {
 725         struct bnxt_tc_info *tc_info = &bp->tc_info;
 726         struct bnxt_tc_flow_node *flow_node;
 727         struct bnxt_tc_flow_stats stats;
 728         int rc;
 729
 730         flow_node = rhashtable_lookup_fast(&tc_info->flow_table,
 731                                            &tc_flow_cmd->cookie,
 732                                            tc_info->flow_ht_params);
 733         if (!flow_node) {
 734                 netdev_info(bp->dev, "Error: no flow_node for cookie %lx",
 735                             tc_flow_cmd->cookie);
 736                 return -1;
 737         }
 738
 739         rc = bnxt_hwrm_cfa_flow_stats_get(bp, flow_node->flow_handle,
 740                                           &flow_node->flow, &stats);
 741         if (rc)
 742                 return rc;
 743
 744         tcf_exts_stats_update(tc_flow_cmd->exts, stats.bytes, stats.packets, 0);
 745         return 0;
 746 }
 747
 748 int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid,
 749                          struct tc_cls_flower_offload *cls_flower)
 750 {
 751         int rc = 0;
 752
 753         if (!is_classid_clsact_ingress(cls_flower->common.classid) ||
 754             cls_flower->common.chain_index)
 755                 return -EOPNOTSUPP;
 756
 757         switch (cls_flower->command) {
 758         case TC_CLSFLOWER_REPLACE:
 759                 rc = bnxt_tc_add_flow(bp, src_fid, cls_flower);
 760                 break;
 761
 762         case TC_CLSFLOWER_DESTROY:
 763                 rc = bnxt_tc_del_flow(bp, cls_flower);
 764                 break;
 765
 766         case TC_CLSFLOWER_STATS:
 767                 rc = bnxt_tc_get_flow_stats(bp, cls_flower);
 768                 break;
 769         }
 770         return rc;
 771 }
 772
 773 static const struct rhashtable_params bnxt_tc_flow_ht_params = {
 774         .head_offset = offsetof(struct bnxt_tc_flow_node, node),
 775         .key_offset = offsetof(struct bnxt_tc_flow_node, cookie),
 776         .key_len = sizeof(((struct bnxt_tc_flow_node *)0)->cookie),
 777         .automatic_shrinking = true
 778 };
 779
 780 static const struct rhashtable_params bnxt_tc_l2_ht_params = {
 781         .head_offset = offsetof(struct bnxt_tc_l2_node, node),
 782         .key_offset = offsetof(struct bnxt_tc_l2_node, key),
 783         .key_len = BNXT_TC_L2_KEY_LEN,
 784         .automatic_shrinking = true
 785 };
 786
 787 /* convert counter width in bits to a mask */
 788 #define mask(width)             ((u64)~0 >> (64 - (width)))
 789
 790 int bnxt_init_tc(struct bnxt *bp)
 791 {
 792         struct bnxt_tc_info *tc_info = &bp->tc_info;
 793         int rc;
 794
 795         if (bp->hwrm_spec_code < 0x10800) {
 796                 netdev_warn(bp->dev,
 797                             "Firmware does not support TC flower offload.\n");
 798                 return -ENOTSUPP;
 799         }
 800         mutex_init(&tc_info->lock);
 801
 802         /* Counter widths are programmed by FW */
 803         tc_info->bytes_mask = mask(36);
 804         tc_info->packets_mask = mask(28);
 805
 806         tc_info->flow_ht_params = bnxt_tc_flow_ht_params;
 807         rc = rhashtable_init(&tc_info->flow_table, &tc_info->flow_ht_params);
 808         if (rc)
 809                 return rc;
 810
 811         tc_info->l2_ht_params = bnxt_tc_l2_ht_params;
 812         rc = rhashtable_init(&tc_info->l2_table, &tc_info->l2_ht_params);
 813         if (rc)
 814                 goto destroy_flow_table;
 815
 816         tc_info->enabled = true;
 817         bp->dev->hw_features |= NETIF_F_HW_TC;
 818         bp->dev->features |= NETIF_F_HW_TC;
 819         return 0;
 820
 821 destroy_flow_table:
 822         rhashtable_destroy(&tc_info->flow_table);
 823         return rc;
 824 }
 825
 826 void bnxt_shutdown_tc(struct bnxt *bp)
 827 {
 828         struct bnxt_tc_info *tc_info = &bp->tc_info;
 829
 830         if (!tc_info->enabled)
 831                 return;
 832
 833         rhashtable_destroy(&tc_info->flow_table);
 834         rhashtable_destroy(&tc_info->l2_table);
 835 }
 836
 837 #else
 838 #endif