drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2017 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/if.h>
  35 #include <linux/if_vlan.h>
  36 #include <linux/if_bridge.h>
  37 #include <linux/rtc.h>
  38 #include <linux/bpf.h>
  39 #include <net/ip.h>
  40 #include <net/tcp.h>
  41 #include <net/udp.h>
  42 #include <net/checksum.h>
  43 #include <net/ip6_checksum.h>
  44 #include <net/udp_tunnel.h>
  45 #include <linux/workqueue.h>
  46 #include <linux/prefetch.h>
  47 #include <linux/cache.h>
  48 #include <linux/log2.h>
  49 #include <linux/aer.h>
  50 #include <linux/bitmap.h>
  51 #include <linux/cpu_rmap.h>
  52 #include <linux/cpumask.h>
  53 #include <net/pkt_cls.h>
  54
  55 #include "bnxt_hsi.h"
  56 #include "bnxt.h"
  57 #include "bnxt_ulp.h"
  58 #include "bnxt_sriov.h"
  59 #include "bnxt_ethtool.h"
  60 #include "bnxt_dcb.h"
  61 #include "bnxt_xdp.h"
  62 #include "bnxt_vfr.h"
  63 #include "bnxt_tc.h"
  64 #include "bnxt_devlink.h"
  65
  66 #define BNXT_TX_TIMEOUT         (5 * HZ)
  67
  68 static const char version[] =
  69         "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
  70
  71 MODULE_LICENSE("GPL");
  72 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  73 MODULE_VERSION(DRV_MODULE_VERSION);
  74
  75 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  76 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  77 #define BNXT_RX_COPY_THRESH 256
  78
  79 #define BNXT_TX_PUSH_THRESH 164
  80
  81 enum board_idx {
  82         BCM57301,
  83         BCM57302,
  84         BCM57304,
  85         BCM57417_NPAR,
  86         BCM58700,
  87         BCM57311,
  88         BCM57312,
  89         BCM57402,
  90         BCM57404,
  91         BCM57406,
  92         BCM57402_NPAR,
  93         BCM57407,
  94         BCM57412,
  95         BCM57414,
  96         BCM57416,
  97         BCM57417,
  98         BCM57412_NPAR,
  99         BCM57314,
 100         BCM57417_SFP,
 101         BCM57416_SFP,
 102         BCM57404_NPAR,
 103         BCM57406_NPAR,
 104         BCM57407_SFP,
 105         BCM57407_NPAR,
 106         BCM57414_NPAR,
 107         BCM57416_NPAR,
 108         BCM57452,
 109         BCM57454,
 110         BCM58802,
 111         BCM58804,
 112         BCM58808,
 113         NETXTREME_E_VF,
 114         NETXTREME_C_VF,
 115         NETXTREME_S_VF,
 116 };
 117
 118 /* indexed by enum above */
 119 static const struct {
 120         char *name;
 121 } board_info[] = {
 122         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
 123         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
 124         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 125         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
 126         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
 127         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
 128         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
 129         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 130         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 131         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 132         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 133         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 134         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 135         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 136         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 137         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 138         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 139         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 140         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 141         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 142         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 143         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 144         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 145         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 146         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 147         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 148         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 149         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 150         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 151         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 152         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 153         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 154         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 155         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 156 };
 157
 158 static const struct pci_device_id bnxt_pci_tbl[] = {
 159         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 160         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 161         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 162         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 163         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 164         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 165         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 166         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 167         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 168         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 169         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 170         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 171         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 172         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 173         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 174         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 175         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 176         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 177         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 178         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 179         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 180         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 181         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 182         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 183         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 184         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 185         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 186         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 187         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 188         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 189         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 190         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 191         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 192         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 193         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 194 #ifdef CONFIG_BNXT_SRIOV
 195         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 196         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 197         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 198         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 199         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 200         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 201         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 202         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 203         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 204 #endif
 205         { 0 }
 206 };
 207
 208 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 209
 210 static const u16 bnxt_vf_req_snif[] = {
 211         HWRM_FUNC_CFG,
 212         HWRM_PORT_PHY_QCFG,
 213         HWRM_CFA_L2_FILTER_ALLOC,
 214 };
 215
 216 static const u16 bnxt_async_events_arr[] = {
 217         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 218         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 219         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 220         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 221         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 222 };
 223
 224 static struct workqueue_struct *bnxt_pf_wq;
 225
 226 static bool bnxt_vf_pciid(enum board_idx idx)
 227 {
 228         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 229                 idx == NETXTREME_S_VF);
 230 }
 231
 232 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 233 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 234 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 235
 236 #define BNXT_CP_DB_REARM(db, raw_cons)                                  \
 237                 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db)
 238
 239 #define BNXT_CP_DB(db, raw_cons)                                        \
 240                 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db)
 241
 242 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 243                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 244
 245 const u16 bnxt_lhint_arr[] = {
 246         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 247         TX_BD_FLAGS_LHINT_512_TO_1023,
 248         TX_BD_FLAGS_LHINT_1024_TO_2047,
 249         TX_BD_FLAGS_LHINT_1024_TO_2047,
 250         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 251         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 252         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 253         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 254         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 255         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 256         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 257         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 258         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 259         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 260         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 261         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 262         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 263         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 264         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 265 };
 266
 267 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 268 {
 269         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 270
 271         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 272                 return 0;
 273
 274         return md_dst->u.port_info.port_id;
 275 }
 276
 277 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 278 {
 279         struct bnxt *bp = netdev_priv(dev);
 280         struct tx_bd *txbd;
 281         struct tx_bd_ext *txbd1;
 282         struct netdev_queue *txq;
 283         int i;
 284         dma_addr_t mapping;
 285         unsigned int length, pad = 0;
 286         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 287         u16 prod, last_frag;
 288         struct pci_dev *pdev = bp->pdev;
 289         struct bnxt_tx_ring_info *txr;
 290         struct bnxt_sw_tx_bd *tx_buf;
 291
 292         i = skb_get_queue_mapping(skb);
 293         if (unlikely(i >= bp->tx_nr_rings)) {
 294                 dev_kfree_skb_any(skb);
 295                 return NETDEV_TX_OK;
 296         }
 297
 298         txq = netdev_get_tx_queue(dev, i);
 299         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 300         prod = txr->tx_prod;
 301
 302         free_size = bnxt_tx_avail(bp, txr);
 303         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 304                 netif_tx_stop_queue(txq);
 305                 return NETDEV_TX_BUSY;
 306         }
 307
 308         length = skb->len;
 309         len = skb_headlen(skb);
 310         last_frag = skb_shinfo(skb)->nr_frags;
 311
 312         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 313
 314         txbd->tx_bd_opaque = prod;
 315
 316         tx_buf = &txr->tx_buf_ring[prod];
 317         tx_buf->skb = skb;
 318         tx_buf->nr_frags = last_frag;
 319
 320         vlan_tag_flags = 0;
 321         cfa_action = bnxt_xmit_get_cfa_action(skb);
 322         if (skb_vlan_tag_present(skb)) {
 323                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 324                                  skb_vlan_tag_get(skb);
 325                 /* Currently supports 8021Q, 8021AD vlan offloads
 326                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 327                  */
 328                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 329                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 330         }
 331
 332         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 333                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 334                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 335                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 336                 void *pdata = tx_push_buf->data;
 337                 u64 *end;
 338                 int j, push_len;
 339
 340                 /* Set COAL_NOW to be ready quickly for the next push */
 341                 tx_push->tx_bd_len_flags_type =
 342                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 343                                         TX_BD_TYPE_LONG_TX_BD |
 344                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 345                                         TX_BD_FLAGS_COAL_NOW |
 346                                         TX_BD_FLAGS_PACKET_END |
 347                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 348
 349                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 350                         tx_push1->tx_bd_hsize_lflags =
 351                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 352                 else
 353                         tx_push1->tx_bd_hsize_lflags = 0;
 354
 355                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 356                 tx_push1->tx_bd_cfa_action =
 357                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 358
 359                 end = pdata + length;
 360                 end = PTR_ALIGN(end, 8) - 1;
 361                 *end = 0;
 362
 363                 skb_copy_from_linear_data(skb, pdata, len);
 364                 pdata += len;
 365                 for (j = 0; j < last_frag; j++) {
 366                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 367                         void *fptr;
 368
 369                         fptr = skb_frag_address_safe(frag);
 370                         if (!fptr)
 371                                 goto normal_tx;
 372
 373                         memcpy(pdata, fptr, skb_frag_size(frag));
 374                         pdata += skb_frag_size(frag);
 375                 }
 376
 377                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 378                 txbd->tx_bd_haddr = txr->data_mapping;
 379                 prod = NEXT_TX(prod);
 380                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 381                 memcpy(txbd, tx_push1, sizeof(*txbd));
 382                 prod = NEXT_TX(prod);
 383                 tx_push->doorbell =
 384                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 385                 txr->tx_prod = prod;
 386
 387                 tx_buf->is_push = 1;
 388                 netdev_tx_sent_queue(txq, skb->len);
 389                 wmb();  /* Sync is_push and byte queue before pushing data */
 390
 391                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 392                 if (push_len > 16) {
 393                         __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16);
 394                         __iowrite32_copy(txr->tx_doorbell + 4, tx_push_buf + 1,
 395                                          (push_len - 16) << 1);
 396                 } else {
 397                         __iowrite64_copy(txr->tx_doorbell, tx_push_buf,
 398                                          push_len);
 399                 }
 400
 401                 goto tx_done;
 402         }
 403
 404 normal_tx:
 405         if (length < BNXT_MIN_PKT_SIZE) {
 406                 pad = BNXT_MIN_PKT_SIZE - length;
 407                 if (skb_pad(skb, pad)) {
 408                         /* SKB already freed. */
 409                         tx_buf->skb = NULL;
 410                         return NETDEV_TX_OK;
 411                 }
 412                 length = BNXT_MIN_PKT_SIZE;
 413         }
 414
 415         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 416
 417         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 418                 dev_kfree_skb_any(skb);
 419                 tx_buf->skb = NULL;
 420                 return NETDEV_TX_OK;
 421         }
 422
 423         dma_unmap_addr_set(tx_buf, mapping, mapping);
 424         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 425                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 426
 427         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 428
 429         prod = NEXT_TX(prod);
 430         txbd1 = (struct tx_bd_ext *)
 431                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 432
 433         txbd1->tx_bd_hsize_lflags = 0;
 434         if (skb_is_gso(skb)) {
 435                 u32 hdr_len;
 436
 437                 if (skb->encapsulation)
 438                         hdr_len = skb_inner_network_offset(skb) +
 439                                 skb_inner_network_header_len(skb) +
 440                                 inner_tcp_hdrlen(skb);
 441                 else
 442                         hdr_len = skb_transport_offset(skb) +
 443                                 tcp_hdrlen(skb);
 444
 445                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 446                                         TX_BD_FLAGS_T_IPID |
 447                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 448                 length = skb_shinfo(skb)->gso_size;
 449                 txbd1->tx_bd_mss = cpu_to_le32(length);
 450                 length += hdr_len;
 451         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 452                 txbd1->tx_bd_hsize_lflags =
 453                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 454                 txbd1->tx_bd_mss = 0;
 455         }
 456
 457         length >>= 9;
 458         flags |= bnxt_lhint_arr[length];
 459         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 460
 461         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 462         txbd1->tx_bd_cfa_action =
 463                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 464         for (i = 0; i < last_frag; i++) {
 465                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 466
 467                 prod = NEXT_TX(prod);
 468                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 469
 470                 len = skb_frag_size(frag);
 471                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 472                                            DMA_TO_DEVICE);
 473
 474                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 475                         goto tx_dma_error;
 476
 477                 tx_buf = &txr->tx_buf_ring[prod];
 478                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 479
 480                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 481
 482                 flags = len << TX_BD_LEN_SHIFT;
 483                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 484         }
 485
 486         flags &= ~TX_BD_LEN;
 487         txbd->tx_bd_len_flags_type =
 488                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 489                             TX_BD_FLAGS_PACKET_END);
 490
 491         netdev_tx_sent_queue(txq, skb->len);
 492
 493         /* Sync BD data before updating doorbell */
 494         wmb();
 495
 496         prod = NEXT_TX(prod);
 497         txr->tx_prod = prod;
 498
 499         if (!skb->xmit_more || netif_xmit_stopped(txq))
 500                 bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod);
 501
 502 tx_done:
 503
 504         mmiowb();
 505
 506         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 507                 if (skb->xmit_more && !tx_buf->is_push)
 508                         bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod);
 509
 510                 netif_tx_stop_queue(txq);
 511
 512                 /* netif_tx_stop_queue() must be done before checking
 513                  * tx index in bnxt_tx_avail() below, because in
 514                  * bnxt_tx_int(), we update tx index before checking for
 515                  * netif_tx_queue_stopped().
 516                  */
 517                 smp_mb();
 518                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 519                         netif_tx_wake_queue(txq);
 520         }
 521         return NETDEV_TX_OK;
 522
 523 tx_dma_error:
 524         last_frag = i;
 525
 526         /* start back at beginning and unmap skb */
 527         prod = txr->tx_prod;
 528         tx_buf = &txr->tx_buf_ring[prod];
 529         tx_buf->skb = NULL;
 530         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 531                          skb_headlen(skb), PCI_DMA_TODEVICE);
 532         prod = NEXT_TX(prod);
 533
 534         /* unmap remaining mapped pages */
 535         for (i = 0; i < last_frag; i++) {
 536                 prod = NEXT_TX(prod);
 537                 tx_buf = &txr->tx_buf_ring[prod];
 538                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 539                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 540                                PCI_DMA_TODEVICE);
 541         }
 542
 543         dev_kfree_skb_any(skb);
 544         return NETDEV_TX_OK;
 545 }
 546
 547 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 548 {
 549         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 550         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 551         u16 cons = txr->tx_cons;
 552         struct pci_dev *pdev = bp->pdev;
 553         int i;
 554         unsigned int tx_bytes = 0;
 555
 556         for (i = 0; i < nr_pkts; i++) {
 557                 struct bnxt_sw_tx_bd *tx_buf;
 558                 struct sk_buff *skb;
 559                 int j, last;
 560
 561                 tx_buf = &txr->tx_buf_ring[cons];
 562                 cons = NEXT_TX(cons);
 563                 skb = tx_buf->skb;
 564                 tx_buf->skb = NULL;
 565
 566                 if (tx_buf->is_push) {
 567                         tx_buf->is_push = 0;
 568                         goto next_tx_int;
 569                 }
 570
 571                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 572                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 573                 last = tx_buf->nr_frags;
 574
 575                 for (j = 0; j < last; j++) {
 576                         cons = NEXT_TX(cons);
 577                         tx_buf = &txr->tx_buf_ring[cons];
 578                         dma_unmap_page(
 579                                 &pdev->dev,
 580                                 dma_unmap_addr(tx_buf, mapping),
 581                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 582                                 PCI_DMA_TODEVICE);
 583                 }
 584
 585 next_tx_int:
 586                 cons = NEXT_TX(cons);
 587
 588                 tx_bytes += skb->len;
 589                 dev_kfree_skb_any(skb);
 590         }
 591
 592         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 593         txr->tx_cons = cons;
 594
 595         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 596          * before checking for netif_tx_queue_stopped().  Without the
 597          * memory barrier, there is a small possibility that bnxt_start_xmit()
 598          * will miss it and cause the queue to be stopped forever.
 599          */
 600         smp_mb();
 601
 602         if (unlikely(netif_tx_queue_stopped(txq)) &&
 603             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 604                 __netif_tx_lock(txq, smp_processor_id());
 605                 if (netif_tx_queue_stopped(txq) &&
 606                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 607                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 608                         netif_tx_wake_queue(txq);
 609                 __netif_tx_unlock(txq);
 610         }
 611 }
 612
 613 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 614                                          gfp_t gfp)
 615 {
 616         struct device *dev = &bp->pdev->dev;
 617         struct page *page;
 618
 619         page = alloc_page(gfp);
 620         if (!page)
 621                 return NULL;
 622
 623         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 624                                       DMA_ATTR_WEAK_ORDERING);
 625         if (dma_mapping_error(dev, *mapping)) {
 626                 __free_page(page);
 627                 return NULL;
 628         }
 629         *mapping += bp->rx_dma_offset;
 630         return page;
 631 }
 632
 633 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 634                                        gfp_t gfp)
 635 {
 636         u8 *data;
 637         struct pci_dev *pdev = bp->pdev;
 638
 639         data = kmalloc(bp->rx_buf_size, gfp);
 640         if (!data)
 641                 return NULL;
 642
 643         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 644                                         bp->rx_buf_use_size, bp->rx_dir,
 645                                         DMA_ATTR_WEAK_ORDERING);
 646
 647         if (dma_mapping_error(&pdev->dev, *mapping)) {
 648                 kfree(data);
 649                 data = NULL;
 650         }
 651         return data;
 652 }
 653
 654 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 655                        u16 prod, gfp_t gfp)
 656 {
 657         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 658         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 659         dma_addr_t mapping;
 660
 661         if (BNXT_RX_PAGE_MODE(bp)) {
 662                 struct page *page = __bnxt_alloc_rx_page(bp, &mapping, gfp);
 663
 664                 if (!page)
 665                         return -ENOMEM;
 666
 667                 rx_buf->data = page;
 668                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 669         } else {
 670                 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 671
 672                 if (!data)
 673                         return -ENOMEM;
 674
 675                 rx_buf->data = data;
 676                 rx_buf->data_ptr = data + bp->rx_offset;
 677         }
 678         rx_buf->mapping = mapping;
 679
 680         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 681         return 0;
 682 }
 683
 684 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 685 {
 686         u16 prod = rxr->rx_prod;
 687         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 688         struct rx_bd *cons_bd, *prod_bd;
 689
 690         prod_rx_buf = &rxr->rx_buf_ring[prod];
 691         cons_rx_buf = &rxr->rx_buf_ring[cons];
 692
 693         prod_rx_buf->data = data;
 694         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 695
 696         prod_rx_buf->mapping = cons_rx_buf->mapping;
 697
 698         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 699         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 700
 701         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 702 }
 703
 704 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 705 {
 706         u16 next, max = rxr->rx_agg_bmap_size;
 707
 708         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 709         if (next >= max)
 710                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 711         return next;
 712 }
 713
 714 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 715                                      struct bnxt_rx_ring_info *rxr,
 716                                      u16 prod, gfp_t gfp)
 717 {
 718         struct rx_bd *rxbd =
 719                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 720         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 721         struct pci_dev *pdev = bp->pdev;
 722         struct page *page;
 723         dma_addr_t mapping;
 724         u16 sw_prod = rxr->rx_sw_agg_prod;
 725         unsigned int offset = 0;
 726
 727         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 728                 page = rxr->rx_page;
 729                 if (!page) {
 730                         page = alloc_page(gfp);
 731                         if (!page)
 732                                 return -ENOMEM;
 733                         rxr->rx_page = page;
 734                         rxr->rx_page_offset = 0;
 735                 }
 736                 offset = rxr->rx_page_offset;
 737                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 738                 if (rxr->rx_page_offset == PAGE_SIZE)
 739                         rxr->rx_page = NULL;
 740                 else
 741                         get_page(page);
 742         } else {
 743                 page = alloc_page(gfp);
 744                 if (!page)
 745                         return -ENOMEM;
 746         }
 747
 748         mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 749                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
 750                                      DMA_ATTR_WEAK_ORDERING);
 751         if (dma_mapping_error(&pdev->dev, mapping)) {
 752                 __free_page(page);
 753                 return -EIO;
 754         }
 755
 756         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 757                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 758
 759         __set_bit(sw_prod, rxr->rx_agg_bmap);
 760         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 761         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 762
 763         rx_agg_buf->page = page;
 764         rx_agg_buf->offset = offset;
 765         rx_agg_buf->mapping = mapping;
 766         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 767         rxbd->rx_bd_opaque = sw_prod;
 768         return 0;
 769 }
 770
 771 static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons,
 772                                    u32 agg_bufs)
 773 {
 774         struct bnxt *bp = bnapi->bp;
 775         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 776         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 777         u16 prod = rxr->rx_agg_prod;
 778         u16 sw_prod = rxr->rx_sw_agg_prod;
 779         u32 i;
 780
 781         for (i = 0; i < agg_bufs; i++) {
 782                 u16 cons;
 783                 struct rx_agg_cmp *agg;
 784                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 785                 struct rx_bd *prod_bd;
 786                 struct page *page;
 787
 788                 agg = (struct rx_agg_cmp *)
 789                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 790                 cons = agg->rx_agg_cmp_opaque;
 791                 __clear_bit(cons, rxr->rx_agg_bmap);
 792
 793                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 794                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 795
 796                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 797                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 798                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 799
 800                 /* It is possible for sw_prod to be equal to cons, so
 801                  * set cons_rx_buf->page to NULL first.
 802                  */
 803                 page = cons_rx_buf->page;
 804                 cons_rx_buf->page = NULL;
 805                 prod_rx_buf->page = page;
 806                 prod_rx_buf->offset = cons_rx_buf->offset;
 807
 808                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 809
 810                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 811
 812                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 813                 prod_bd->rx_bd_opaque = sw_prod;
 814
 815                 prod = NEXT_RX_AGG(prod);
 816                 sw_prod = NEXT_RX_AGG(sw_prod);
 817                 cp_cons = NEXT_CMP(cp_cons);
 818         }
 819         rxr->rx_agg_prod = prod;
 820         rxr->rx_sw_agg_prod = sw_prod;
 821 }
 822
 823 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 824                                         struct bnxt_rx_ring_info *rxr,
 825                                         u16 cons, void *data, u8 *data_ptr,
 826                                         dma_addr_t dma_addr,
 827                                         unsigned int offset_and_len)
 828 {
 829         unsigned int payload = offset_and_len >> 16;
 830         unsigned int len = offset_and_len & 0xffff;
 831         struct skb_frag_struct *frag;
 832         struct page *page = data;
 833         u16 prod = rxr->rx_prod;
 834         struct sk_buff *skb;
 835         int off, err;
 836
 837         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 838         if (unlikely(err)) {
 839                 bnxt_reuse_rx_data(rxr, cons, data);
 840                 return NULL;
 841         }
 842         dma_addr -= bp->rx_dma_offset;
 843         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 844                              DMA_ATTR_WEAK_ORDERING);
 845
 846         if (unlikely(!payload))
 847                 payload = eth_get_headlen(data_ptr, len);
 848
 849         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
 850         if (!skb) {
 851                 __free_page(page);
 852                 return NULL;
 853         }
 854
 855         off = (void *)data_ptr - page_address(page);
 856         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
 857         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
 858                payload + NET_IP_ALIGN);
 859
 860         frag = &skb_shinfo(skb)->frags[0];
 861         skb_frag_size_sub(frag, payload);
 862         frag->page_offset += payload;
 863         skb->data_len -= payload;
 864         skb->tail += payload;
 865
 866         return skb;
 867 }
 868
 869 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 870                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 871                                    void *data, u8 *data_ptr,
 872                                    dma_addr_t dma_addr,
 873                                    unsigned int offset_and_len)
 874 {
 875         u16 prod = rxr->rx_prod;
 876         struct sk_buff *skb;
 877         int err;
 878
 879         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 880         if (unlikely(err)) {
 881                 bnxt_reuse_rx_data(rxr, cons, data);
 882                 return NULL;
 883         }
 884
 885         skb = build_skb(data, 0);
 886         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 887                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
 888         if (!skb) {
 889                 kfree(data);
 890                 return NULL;
 891         }
 892
 893         skb_reserve(skb, bp->rx_offset);
 894         skb_put(skb, offset_and_len & 0xffff);
 895         return skb;
 896 }
 897
 898 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi,
 899                                      struct sk_buff *skb, u16 cp_cons,
 900                                      u32 agg_bufs)
 901 {
 902         struct pci_dev *pdev = bp->pdev;
 903         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 904         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 905         u16 prod = rxr->rx_agg_prod;
 906         u32 i;
 907
 908         for (i = 0; i < agg_bufs; i++) {
 909                 u16 cons, frag_len;
 910                 struct rx_agg_cmp *agg;
 911                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
 912                 struct page *page;
 913                 dma_addr_t mapping;
 914
 915                 agg = (struct rx_agg_cmp *)
 916                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 917                 cons = agg->rx_agg_cmp_opaque;
 918                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
 919                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
 920
 921                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 922                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
 923                                    cons_rx_buf->offset, frag_len);
 924                 __clear_bit(cons, rxr->rx_agg_bmap);
 925
 926                 /* It is possible for bnxt_alloc_rx_page() to allocate
 927                  * a sw_prod index that equals the cons index, so we
 928                  * need to clear the cons entry now.
 929                  */
 930                 mapping = cons_rx_buf->mapping;
 931                 page = cons_rx_buf->page;
 932                 cons_rx_buf->page = NULL;
 933
 934                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
 935                         struct skb_shared_info *shinfo;
 936                         unsigned int nr_frags;
 937
 938                         shinfo = skb_shinfo(skb);
 939                         nr_frags = --shinfo->nr_frags;
 940                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
 941
 942                         dev_kfree_skb(skb);
 943
 944                         cons_rx_buf->page = page;
 945
 946                         /* Update prod since possibly some pages have been
 947                          * allocated already.
 948                          */
 949                         rxr->rx_agg_prod = prod;
 950                         bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i);
 951                         return NULL;
 952                 }
 953
 954                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
 955                                      PCI_DMA_FROMDEVICE,
 956                                      DMA_ATTR_WEAK_ORDERING);
 957
 958                 skb->data_len += frag_len;
 959                 skb->len += frag_len;
 960                 skb->truesize += PAGE_SIZE;
 961
 962                 prod = NEXT_RX_AGG(prod);
 963                 cp_cons = NEXT_CMP(cp_cons);
 964         }
 965         rxr->rx_agg_prod = prod;
 966         return skb;
 967 }
 968
 969 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 970                                u8 agg_bufs, u32 *raw_cons)
 971 {
 972         u16 last;
 973         struct rx_agg_cmp *agg;
 974
 975         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
 976         last = RING_CMP(*raw_cons);
 977         agg = (struct rx_agg_cmp *)
 978                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
 979         return RX_AGG_CMP_VALID(agg, *raw_cons);
 980 }
 981
 982 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
 983                                             unsigned int len,
 984                                             dma_addr_t mapping)
 985 {
 986         struct bnxt *bp = bnapi->bp;
 987         struct pci_dev *pdev = bp->pdev;
 988         struct sk_buff *skb;
 989
 990         skb = napi_alloc_skb(&bnapi->napi, len);
 991         if (!skb)
 992                 return NULL;
 993
 994         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
 995                                 bp->rx_dir);
 996
 997         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
 998                len + NET_IP_ALIGN);
 999
1000         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1001                                    bp->rx_dir);
1002
1003         skb_put(skb, len);
1004         return skb;
1005 }
1006
1007 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_napi *bnapi,
1008                            u32 *raw_cons, void *cmp)
1009 {
1010         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1011         struct rx_cmp *rxcmp = cmp;
1012         u32 tmp_raw_cons = *raw_cons;
1013         u8 cmp_type, agg_bufs = 0;
1014
1015         cmp_type = RX_CMP_TYPE(rxcmp);
1016
1017         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1018                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1019                             RX_CMP_AGG_BUFS) >>
1020                            RX_CMP_AGG_BUFS_SHIFT;
1021         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1022                 struct rx_tpa_end_cmp *tpa_end = cmp;
1023
1024                 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1025                             RX_TPA_END_CMP_AGG_BUFS) >>
1026                            RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1027         }
1028
1029         if (agg_bufs) {
1030                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1031                         return -EBUSY;
1032         }
1033         *raw_cons = tmp_raw_cons;
1034         return 0;
1035 }
1036
1037 static void bnxt_queue_sp_work(struct bnxt *bp)
1038 {
1039         if (BNXT_PF(bp))
1040                 queue_work(bnxt_pf_wq, &bp->sp_task);
1041         else
1042                 schedule_work(&bp->sp_task);
1043 }
1044
1045 static void bnxt_cancel_sp_work(struct bnxt *bp)
1046 {
1047         if (BNXT_PF(bp))
1048                 flush_workqueue(bnxt_pf_wq);
1049         else
1050                 cancel_work_sync(&bp->sp_task);
1051 }
1052
1053 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1054 {
1055         if (!rxr->bnapi->in_reset) {
1056                 rxr->bnapi->in_reset = true;
1057                 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1058                 bnxt_queue_sp_work(bp);
1059         }
1060         rxr->rx_next_cons = 0xffff;
1061 }
1062
1063 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1064                            struct rx_tpa_start_cmp *tpa_start,
1065                            struct rx_tpa_start_cmp_ext *tpa_start1)
1066 {
1067         u8 agg_id = TPA_START_AGG_ID(tpa_start);
1068         u16 cons, prod;
1069         struct bnxt_tpa_info *tpa_info;
1070         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1071         struct rx_bd *prod_bd;
1072         dma_addr_t mapping;
1073
1074         cons = tpa_start->rx_tpa_start_cmp_opaque;
1075         prod = rxr->rx_prod;
1076         cons_rx_buf = &rxr->rx_buf_ring[cons];
1077         prod_rx_buf = &rxr->rx_buf_ring[prod];
1078         tpa_info = &rxr->rx_tpa[agg_id];
1079
1080         if (unlikely(cons != rxr->rx_next_cons)) {
1081                 bnxt_sched_reset(bp, rxr);
1082                 return;
1083         }
1084         /* Store cfa_code in tpa_info to use in tpa_end
1085          * completion processing.
1086          */
1087         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1088         prod_rx_buf->data = tpa_info->data;
1089         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1090
1091         mapping = tpa_info->mapping;
1092         prod_rx_buf->mapping = mapping;
1093
1094         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1095
1096         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1097
1098         tpa_info->data = cons_rx_buf->data;
1099         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1100         cons_rx_buf->data = NULL;
1101         tpa_info->mapping = cons_rx_buf->mapping;
1102
1103         tpa_info->len =
1104                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1105                                 RX_TPA_START_CMP_LEN_SHIFT;
1106         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1107                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1108
1109                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1110                 tpa_info->gso_type = SKB_GSO_TCPV4;
1111                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1112                 if (hash_type == 3)
1113                         tpa_info->gso_type = SKB_GSO_TCPV6;
1114                 tpa_info->rss_hash =
1115                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1116         } else {
1117                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1118                 tpa_info->gso_type = 0;
1119                 if (netif_msg_rx_err(bp))
1120                         netdev_warn(bp->dev, "TPA packet without valid hash\n");
1121         }
1122         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1123         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1124         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1125
1126         rxr->rx_prod = NEXT_RX(prod);
1127         cons = NEXT_RX(cons);
1128         rxr->rx_next_cons = NEXT_RX(cons);
1129         cons_rx_buf = &rxr->rx_buf_ring[cons];
1130
1131         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1132         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1133         cons_rx_buf->data = NULL;
1134 }
1135
1136 static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi,
1137                            u16 cp_cons, u32 agg_bufs)
1138 {
1139         if (agg_bufs)
1140                 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
1141 }
1142
1143 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1144                                            int payload_off, int tcp_ts,
1145                                            struct sk_buff *skb)
1146 {
1147 #ifdef CONFIG_INET
1148         struct tcphdr *th;
1149         int len, nw_off;
1150         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1151         u32 hdr_info = tpa_info->hdr_info;
1152         bool loopback = false;
1153
1154         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1155         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1156         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1157
1158         /* If the packet is an internal loopback packet, the offsets will
1159          * have an extra 4 bytes.
1160          */
1161         if (inner_mac_off == 4) {
1162                 loopback = true;
1163         } else if (inner_mac_off > 4) {
1164                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1165                                             ETH_HLEN - 2));
1166
1167                 /* We only support inner iPv4/ipv6.  If we don't see the
1168                  * correct protocol ID, it must be a loopback packet where
1169                  * the offsets are off by 4.
1170                  */
1171                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1172                         loopback = true;
1173         }
1174         if (loopback) {
1175                 /* internal loopback packet, subtract all offsets by 4 */
1176                 inner_ip_off -= 4;
1177                 inner_mac_off -= 4;
1178                 outer_ip_off -= 4;
1179         }
1180
1181         nw_off = inner_ip_off - ETH_HLEN;
1182         skb_set_network_header(skb, nw_off);
1183         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1184                 struct ipv6hdr *iph = ipv6_hdr(skb);
1185
1186                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1187                 len = skb->len - skb_transport_offset(skb);
1188                 th = tcp_hdr(skb);
1189                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1190         } else {
1191                 struct iphdr *iph = ip_hdr(skb);
1192
1193                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1194                 len = skb->len - skb_transport_offset(skb);
1195                 th = tcp_hdr(skb);
1196                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1197         }
1198
1199         if (inner_mac_off) { /* tunnel */
1200                 struct udphdr *uh = NULL;
1201                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1202                                             ETH_HLEN - 2));
1203
1204                 if (proto == htons(ETH_P_IP)) {
1205                         struct iphdr *iph = (struct iphdr *)skb->data;
1206
1207                         if (iph->protocol == IPPROTO_UDP)
1208                                 uh = (struct udphdr *)(iph + 1);
1209                 } else {
1210                         struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1211
1212                         if (iph->nexthdr == IPPROTO_UDP)
1213                                 uh = (struct udphdr *)(iph + 1);
1214                 }
1215                 if (uh) {
1216                         if (uh->check)
1217                                 skb_shinfo(skb)->gso_type |=
1218                                         SKB_GSO_UDP_TUNNEL_CSUM;
1219                         else
1220                                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1221                 }
1222         }
1223 #endif
1224         return skb;
1225 }
1226
1227 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1228 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1229
1230 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1231                                            int payload_off, int tcp_ts,
1232                                            struct sk_buff *skb)
1233 {
1234 #ifdef CONFIG_INET
1235         struct tcphdr *th;
1236         int len, nw_off, tcp_opt_len = 0;
1237
1238         if (tcp_ts)
1239                 tcp_opt_len = 12;
1240
1241         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1242                 struct iphdr *iph;
1243
1244                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1245                          ETH_HLEN;
1246                 skb_set_network_header(skb, nw_off);
1247                 iph = ip_hdr(skb);
1248                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1249                 len = skb->len - skb_transport_offset(skb);
1250                 th = tcp_hdr(skb);
1251                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1252         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1253                 struct ipv6hdr *iph;
1254
1255                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1256                          ETH_HLEN;
1257                 skb_set_network_header(skb, nw_off);
1258                 iph = ipv6_hdr(skb);
1259                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1260                 len = skb->len - skb_transport_offset(skb);
1261                 th = tcp_hdr(skb);
1262                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1263         } else {
1264                 dev_kfree_skb_any(skb);
1265                 return NULL;
1266         }
1267
1268         if (nw_off) { /* tunnel */
1269                 struct udphdr *uh = NULL;
1270
1271                 if (skb->protocol == htons(ETH_P_IP)) {
1272                         struct iphdr *iph = (struct iphdr *)skb->data;
1273
1274                         if (iph->protocol == IPPROTO_UDP)
1275                                 uh = (struct udphdr *)(iph + 1);
1276                 } else {
1277                         struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1278
1279                         if (iph->nexthdr == IPPROTO_UDP)
1280                                 uh = (struct udphdr *)(iph + 1);
1281                 }
1282                 if (uh) {
1283                         if (uh->check)
1284                                 skb_shinfo(skb)->gso_type |=
1285                                         SKB_GSO_UDP_TUNNEL_CSUM;
1286                         else
1287                                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1288                 }
1289         }
1290 #endif
1291         return skb;
1292 }
1293
1294 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1295                                            struct bnxt_tpa_info *tpa_info,
1296                                            struct rx_tpa_end_cmp *tpa_end,
1297                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1298                                            struct sk_buff *skb)
1299 {
1300 #ifdef CONFIG_INET
1301         int payload_off;
1302         u16 segs;
1303
1304         segs = TPA_END_TPA_SEGS(tpa_end);
1305         if (segs == 1)
1306                 return skb;
1307
1308         NAPI_GRO_CB(skb)->count = segs;
1309         skb_shinfo(skb)->gso_size =
1310                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1311         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1312         payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1313                        RX_TPA_END_CMP_PAYLOAD_OFFSET) >>
1314                       RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT;
1315         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1316         if (likely(skb))
1317                 tcp_gro_complete(skb);
1318 #endif
1319         return skb;
1320 }
1321
1322 /* Given the cfa_code of a received packet determine which
1323  * netdev (vf-rep or PF) the packet is destined to.
1324  */
1325 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1326 {
1327         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1328
1329         /* if vf-rep dev is NULL, the must belongs to the PF */
1330         return dev ? dev : bp->dev;
1331 }
1332
1333 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1334                                            struct bnxt_napi *bnapi,
1335                                            u32 *raw_cons,
1336                                            struct rx_tpa_end_cmp *tpa_end,
1337                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1338                                            u8 *event)
1339 {
1340         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1341         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1342         u8 agg_id = TPA_END_AGG_ID(tpa_end);
1343         u8 *data_ptr, agg_bufs;
1344         u16 cp_cons = RING_CMP(*raw_cons);
1345         unsigned int len;
1346         struct bnxt_tpa_info *tpa_info;
1347         dma_addr_t mapping;
1348         struct sk_buff *skb;
1349         void *data;
1350
1351         if (unlikely(bnapi->in_reset)) {
1352                 int rc = bnxt_discard_rx(bp, bnapi, raw_cons, tpa_end);
1353
1354                 if (rc < 0)
1355                         return ERR_PTR(-EBUSY);
1356                 return NULL;
1357         }
1358
1359         tpa_info = &rxr->rx_tpa[agg_id];
1360         data = tpa_info->data;
1361         data_ptr = tpa_info->data_ptr;
1362         prefetch(data_ptr);
1363         len = tpa_info->len;
1364         mapping = tpa_info->mapping;
1365
1366         agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1367                     RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1368
1369         if (agg_bufs) {
1370                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1371                         return ERR_PTR(-EBUSY);
1372
1373                 *event |= BNXT_AGG_EVENT;
1374                 cp_cons = NEXT_CMP(cp_cons);
1375         }
1376
1377         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1378                 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1379                 if (agg_bufs > MAX_SKB_FRAGS)
1380                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1381                                     agg_bufs, (int)MAX_SKB_FRAGS);
1382                 return NULL;
1383         }
1384
1385         if (len <= bp->rx_copy_thresh) {
1386                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1387                 if (!skb) {
1388                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1389                         return NULL;
1390                 }
1391         } else {
1392                 u8 *new_data;
1393                 dma_addr_t new_mapping;
1394
1395                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1396                 if (!new_data) {
1397                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1398                         return NULL;
1399                 }
1400
1401                 tpa_info->data = new_data;
1402                 tpa_info->data_ptr = new_data + bp->rx_offset;
1403                 tpa_info->mapping = new_mapping;
1404
1405                 skb = build_skb(data, 0);
1406                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1407                                        bp->rx_buf_use_size, bp->rx_dir,
1408                                        DMA_ATTR_WEAK_ORDERING);
1409
1410                 if (!skb) {
1411                         kfree(data);
1412                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1413                         return NULL;
1414                 }
1415                 skb_reserve(skb, bp->rx_offset);
1416                 skb_put(skb, len);
1417         }
1418
1419         if (agg_bufs) {
1420                 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1421                 if (!skb) {
1422                         /* Page reuse already handled by bnxt_rx_pages(). */
1423                         return NULL;
1424                 }
1425         }
1426
1427         skb->protocol =
1428                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1429
1430         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1431                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1432
1433         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1434             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1435                 u16 vlan_proto = tpa_info->metadata >>
1436                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1437                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_VID_MASK;
1438
1439                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1440         }
1441
1442         skb_checksum_none_assert(skb);
1443         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1444                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1445                 skb->csum_level =
1446                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1447         }
1448
1449         if (TPA_END_GRO(tpa_end))
1450                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1451
1452         return skb;
1453 }
1454
1455 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1456                              struct sk_buff *skb)
1457 {
1458         if (skb->dev != bp->dev) {
1459                 /* this packet belongs to a vf-rep */
1460                 bnxt_vf_rep_rx(bp, skb);
1461                 return;
1462         }
1463         skb_record_rx_queue(skb, bnapi->index);
1464         napi_gro_receive(&bnapi->napi, skb);
1465 }
1466
1467 /* returns the following:
1468  * 1       - 1 packet successfully received
1469  * 0       - successful TPA_START, packet not completed yet
1470  * -EBUSY  - completion ring does not have all the agg buffers yet
1471  * -ENOMEM - packet aborted due to out of memory
1472  * -EIO    - packet aborted due to hw error indicated in BD
1473  */
1474 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
1475                        u8 *event)
1476 {
1477         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1478         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1479         struct net_device *dev = bp->dev;
1480         struct rx_cmp *rxcmp;
1481         struct rx_cmp_ext *rxcmp1;
1482         u32 tmp_raw_cons = *raw_cons;
1483         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1484         struct bnxt_sw_rx_bd *rx_buf;
1485         unsigned int len;
1486         u8 *data_ptr, agg_bufs, cmp_type;
1487         dma_addr_t dma_addr;
1488         struct sk_buff *skb;
1489         void *data;
1490         int rc = 0;
1491         u32 misc;
1492
1493         rxcmp = (struct rx_cmp *)
1494                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1495
1496         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1497         cp_cons = RING_CMP(tmp_raw_cons);
1498         rxcmp1 = (struct rx_cmp_ext *)
1499                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1500
1501         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1502                 return -EBUSY;
1503
1504         cmp_type = RX_CMP_TYPE(rxcmp);
1505
1506         prod = rxr->rx_prod;
1507
1508         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1509                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1510                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1511
1512                 *event |= BNXT_RX_EVENT;
1513                 goto next_rx_no_prod;
1514
1515         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1516                 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons,
1517                                    (struct rx_tpa_end_cmp *)rxcmp,
1518                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1519
1520                 if (IS_ERR(skb))
1521                         return -EBUSY;
1522
1523                 rc = -ENOMEM;
1524                 if (likely(skb)) {
1525                         bnxt_deliver_skb(bp, bnapi, skb);
1526                         rc = 1;
1527                 }
1528                 *event |= BNXT_RX_EVENT;
1529                 goto next_rx_no_prod;
1530         }
1531
1532         cons = rxcmp->rx_cmp_opaque;
1533         rx_buf = &rxr->rx_buf_ring[cons];
1534         data = rx_buf->data;
1535         data_ptr = rx_buf->data_ptr;
1536         if (unlikely(cons != rxr->rx_next_cons)) {
1537                 int rc1 = bnxt_discard_rx(bp, bnapi, raw_cons, rxcmp);
1538
1539                 bnxt_sched_reset(bp, rxr);
1540                 return rc1;
1541         }
1542         prefetch(data_ptr);
1543
1544         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1545         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1546
1547         if (agg_bufs) {
1548                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1549                         return -EBUSY;
1550
1551                 cp_cons = NEXT_CMP(cp_cons);
1552                 *event |= BNXT_AGG_EVENT;
1553         }
1554         *event |= BNXT_RX_EVENT;
1555
1556         rx_buf->data = NULL;
1557         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1558                 bnxt_reuse_rx_data(rxr, cons, data);
1559                 if (agg_bufs)
1560                         bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
1561
1562                 rc = -EIO;
1563                 goto next_rx;
1564         }
1565
1566         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1567         dma_addr = rx_buf->mapping;
1568
1569         if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1570                 rc = 1;
1571                 goto next_rx;
1572         }
1573
1574         if (len <= bp->rx_copy_thresh) {
1575                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1576                 bnxt_reuse_rx_data(rxr, cons, data);
1577                 if (!skb) {
1578                         rc = -ENOMEM;
1579                         goto next_rx;
1580                 }
1581         } else {
1582                 u32 payload;
1583
1584                 if (rx_buf->data_ptr == data_ptr)
1585                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1586                 else
1587                         payload = 0;
1588                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1589                                       payload | len);
1590                 if (!skb) {
1591                         rc = -ENOMEM;
1592                         goto next_rx;
1593                 }
1594         }
1595
1596         if (agg_bufs) {
1597                 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1598                 if (!skb) {
1599                         rc = -ENOMEM;
1600                         goto next_rx;
1601                 }
1602         }
1603
1604         if (RX_CMP_HASH_VALID(rxcmp)) {
1605                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1606                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1607
1608                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1609                 if (hash_type != 1 && hash_type != 3)
1610                         type = PKT_HASH_TYPE_L3;
1611                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1612         }
1613
1614         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1615         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1616
1617         if ((rxcmp1->rx_cmp_flags2 &
1618              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1619             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1620                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1621                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_VID_MASK;
1622                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1623
1624                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1625         }
1626
1627         skb_checksum_none_assert(skb);
1628         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1629                 if (dev->features & NETIF_F_RXCSUM) {
1630                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1631                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1632                 }
1633         } else {
1634                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1635                         if (dev->features & NETIF_F_RXCSUM)
1636                                 cpr->rx_l4_csum_errors++;
1637                 }
1638         }
1639
1640         bnxt_deliver_skb(bp, bnapi, skb);
1641         rc = 1;
1642
1643 next_rx:
1644         rxr->rx_prod = NEXT_RX(prod);
1645         rxr->rx_next_cons = NEXT_RX(cons);
1646
1647 next_rx_no_prod:
1648         *raw_cons = tmp_raw_cons;
1649
1650         return rc;
1651 }
1652
1653 /* In netpoll mode, if we are using a combined completion ring, we need to
1654  * discard the rx packets and recycle the buffers.
1655  */
1656 static int bnxt_force_rx_discard(struct bnxt *bp, struct bnxt_napi *bnapi,
1657                                  u32 *raw_cons, u8 *event)
1658 {
1659         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1660         u32 tmp_raw_cons = *raw_cons;
1661         struct rx_cmp_ext *rxcmp1;
1662         struct rx_cmp *rxcmp;
1663         u16 cp_cons;
1664         u8 cmp_type;
1665
1666         cp_cons = RING_CMP(tmp_raw_cons);
1667         rxcmp = (struct rx_cmp *)
1668                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1669
1670         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1671         cp_cons = RING_CMP(tmp_raw_cons);
1672         rxcmp1 = (struct rx_cmp_ext *)
1673                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1674
1675         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1676                 return -EBUSY;
1677
1678         cmp_type = RX_CMP_TYPE(rxcmp);
1679         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1680                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1681                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1682         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1683                 struct rx_tpa_end_cmp_ext *tpa_end1;
1684
1685                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1686                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1687                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1688         }
1689         return bnxt_rx_pkt(bp, bnapi, raw_cons, event);
1690 }
1691
1692 #define BNXT_GET_EVENT_PORT(data)       \
1693         ((data) &                       \
1694          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1695
1696 static int bnxt_async_event_process(struct bnxt *bp,
1697                                     struct hwrm_async_event_cmpl *cmpl)
1698 {
1699         u16 event_id = le16_to_cpu(cmpl->event_id);
1700
1701         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1702         switch (event_id) {
1703         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1704                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1705                 struct bnxt_link_info *link_info = &bp->link_info;
1706
1707                 if (BNXT_VF(bp))
1708                         goto async_event_process_exit;
1709                 if (data1 & 0x20000) {
1710                         u16 fw_speed = link_info->force_link_speed;
1711                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1712
1713                         netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1714                                     speed);
1715                 }
1716                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1717                 /* fall thru */
1718         }
1719         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1720                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1721                 break;
1722         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1723                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1724                 break;
1725         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1726                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1727                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1728
1729                 if (BNXT_VF(bp))
1730                         break;
1731
1732                 if (bp->pf.port_id != port_id)
1733                         break;
1734
1735                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1736                 break;
1737         }
1738         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
1739                 if (BNXT_PF(bp))
1740                         goto async_event_process_exit;
1741                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
1742                 break;
1743         default:
1744                 goto async_event_process_exit;
1745         }
1746         bnxt_queue_sp_work(bp);
1747 async_event_process_exit:
1748         bnxt_ulp_async_events(bp, cmpl);
1749         return 0;
1750 }
1751
1752 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1753 {
1754         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1755         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1756         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1757                                 (struct hwrm_fwd_req_cmpl *)txcmp;
1758
1759         switch (cmpl_type) {
1760         case CMPL_BASE_TYPE_HWRM_DONE:
1761                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1762                 if (seq_id == bp->hwrm_intr_seq_id)
1763                         bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID;
1764                 else
1765                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1766                 break;
1767
1768         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1769                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1770
1771                 if ((vf_id < bp->pf.first_vf_id) ||
1772                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1773                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1774                                    vf_id);
1775                         return -EINVAL;
1776                 }
1777
1778                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1779                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
1780                 bnxt_queue_sp_work(bp);
1781                 break;
1782
1783         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1784                 bnxt_async_event_process(bp,
1785                                          (struct hwrm_async_event_cmpl *)txcmp);
1786
1787         default:
1788                 break;
1789         }
1790
1791         return 0;
1792 }
1793
1794 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
1795 {
1796         struct bnxt_napi *bnapi = dev_instance;
1797         struct bnxt *bp = bnapi->bp;
1798         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1799         u32 cons = RING_CMP(cpr->cp_raw_cons);
1800
1801         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1802         napi_schedule(&bnapi->napi);
1803         return IRQ_HANDLED;
1804 }
1805
1806 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
1807 {
1808         u32 raw_cons = cpr->cp_raw_cons;
1809         u16 cons = RING_CMP(raw_cons);
1810         struct tx_cmp *txcmp;
1811
1812         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1813
1814         return TX_CMP_VALID(txcmp, raw_cons);
1815 }
1816
1817 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
1818 {
1819         struct bnxt_napi *bnapi = dev_instance;
1820         struct bnxt *bp = bnapi->bp;
1821         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1822         u32 cons = RING_CMP(cpr->cp_raw_cons);
1823         u32 int_status;
1824
1825         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1826
1827         if (!bnxt_has_work(bp, cpr)) {
1828                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
1829                 /* return if erroneous interrupt */
1830                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
1831                         return IRQ_NONE;
1832         }
1833
1834         /* disable ring IRQ */
1835         BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell);
1836
1837         /* Return here if interrupt is shared and is disabled. */
1838         if (unlikely(atomic_read(&bp->intr_sem) != 0))
1839                 return IRQ_HANDLED;
1840
1841         napi_schedule(&bnapi->napi);
1842         return IRQ_HANDLED;
1843 }
1844
1845 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
1846 {
1847         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1848         u32 raw_cons = cpr->cp_raw_cons;
1849         u32 cons;
1850         int tx_pkts = 0;
1851         int rx_pkts = 0;
1852         u8 event = 0;
1853         struct tx_cmp *txcmp;
1854
1855         while (1) {
1856                 int rc;
1857
1858                 cons = RING_CMP(raw_cons);
1859                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1860
1861                 if (!TX_CMP_VALID(txcmp, raw_cons))
1862                         break;
1863
1864                 /* The valid test of the entry must be done first before
1865                  * reading any further.
1866                  */
1867                 dma_rmb();
1868                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
1869                         tx_pkts++;
1870                         /* return full budget so NAPI will complete. */
1871                         if (unlikely(tx_pkts > bp->tx_wake_thresh))
1872                                 rx_pkts = budget;
1873                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1874                         if (likely(budget))
1875                                 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event);
1876                         else
1877                                 rc = bnxt_force_rx_discard(bp, bnapi, &raw_cons,
1878                                                            &event);
1879                         if (likely(rc >= 0))
1880                                 rx_pkts += rc;
1881                         /* Increment rx_pkts when rc is -ENOMEM to count towards
1882                          * the NAPI budget.  Otherwise, we may potentially loop
1883                          * here forever if we consistently cannot allocate
1884                          * buffers.
1885                          */
1886                         else if (rc == -ENOMEM && budget)
1887                                 rx_pkts++;
1888                         else if (rc == -EBUSY)  /* partial completion */
1889                                 break;
1890                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
1891                                      CMPL_BASE_TYPE_HWRM_DONE) ||
1892                                     (TX_CMP_TYPE(txcmp) ==
1893                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
1894                                     (TX_CMP_TYPE(txcmp) ==
1895                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
1896                         bnxt_hwrm_handler(bp, txcmp);
1897                 }
1898                 raw_cons = NEXT_RAW_CMP(raw_cons);
1899
1900                 if (rx_pkts == budget)
1901                         break;
1902         }
1903
1904         if (event & BNXT_TX_EVENT) {
1905                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
1906                 void __iomem *db = txr->tx_doorbell;
1907                 u16 prod = txr->tx_prod;
1908
1909                 /* Sync BD data before updating doorbell */
1910                 wmb();
1911
1912                 bnxt_db_write(bp, db, DB_KEY_TX | prod);
1913         }
1914
1915         cpr->cp_raw_cons = raw_cons;
1916         /* ACK completion ring before freeing tx ring and producing new
1917          * buffers in rx/agg rings to prevent overflowing the completion
1918          * ring.
1919          */
1920         BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
1921
1922         if (tx_pkts)
1923                 bnapi->tx_int(bp, bnapi, tx_pkts);
1924
1925         if (event & BNXT_RX_EVENT) {
1926                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1927
1928                 bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod);
1929                 if (event & BNXT_AGG_EVENT)
1930                         bnxt_db_write(bp, rxr->rx_agg_doorbell,
1931                                       DB_KEY_RX | rxr->rx_agg_prod);
1932         }
1933         return rx_pkts;
1934 }
1935
1936 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
1937 {
1938         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1939         struct bnxt *bp = bnapi->bp;
1940         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1941         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1942         struct tx_cmp *txcmp;
1943         struct rx_cmp_ext *rxcmp1;
1944         u32 cp_cons, tmp_raw_cons;
1945         u32 raw_cons = cpr->cp_raw_cons;
1946         u32 rx_pkts = 0;
1947         u8 event = 0;
1948
1949         while (1) {
1950                 int rc;
1951
1952                 cp_cons = RING_CMP(raw_cons);
1953                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1954
1955                 if (!TX_CMP_VALID(txcmp, raw_cons))
1956                         break;
1957
1958                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1959                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
1960                         cp_cons = RING_CMP(tmp_raw_cons);
1961                         rxcmp1 = (struct rx_cmp_ext *)
1962                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1963
1964                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1965                                 break;
1966
1967                         /* force an error to recycle the buffer */
1968                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1969                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1970
1971                         rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event);
1972                         if (likely(rc == -EIO) && budget)
1973                                 rx_pkts++;
1974                         else if (rc == -EBUSY)  /* partial completion */
1975                                 break;
1976                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
1977                                     CMPL_BASE_TYPE_HWRM_DONE)) {
1978                         bnxt_hwrm_handler(bp, txcmp);
1979                 } else {
1980                         netdev_err(bp->dev,
1981                                    "Invalid completion received on special ring\n");
1982                 }
1983                 raw_cons = NEXT_RAW_CMP(raw_cons);
1984
1985                 if (rx_pkts == budget)
1986                         break;
1987         }
1988
1989         cpr->cp_raw_cons = raw_cons;
1990         BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
1991         bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod);
1992
1993         if (event & BNXT_AGG_EVENT)
1994                 bnxt_db_write(bp, rxr->rx_agg_doorbell,
1995                               DB_KEY_RX | rxr->rx_agg_prod);
1996
1997         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
1998                 napi_complete_done(napi, rx_pkts);
1999                 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
2000         }
2001         return rx_pkts;
2002 }
2003
2004 static int bnxt_poll(struct napi_struct *napi, int budget)
2005 {
2006         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2007         struct bnxt *bp = bnapi->bp;
2008         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2009         int work_done = 0;
2010
2011         while (1) {
2012                 work_done += bnxt_poll_work(bp, bnapi, budget - work_done);
2013
2014                 if (work_done >= budget)
2015                         break;
2016
2017                 if (!bnxt_has_work(bp, cpr)) {
2018                         if (napi_complete_done(napi, work_done))
2019                                 BNXT_CP_DB_REARM(cpr->cp_doorbell,
2020                                                  cpr->cp_raw_cons);
2021                         break;
2022                 }
2023         }
2024         mmiowb();
2025         return work_done;
2026 }
2027
2028 static void bnxt_free_tx_skbs(struct bnxt *bp)
2029 {
2030         int i, max_idx;
2031         struct pci_dev *pdev = bp->pdev;
2032
2033         if (!bp->tx_ring)
2034                 return;
2035
2036         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2037         for (i = 0; i < bp->tx_nr_rings; i++) {
2038                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2039                 int j;
2040
2041                 for (j = 0; j < max_idx;) {
2042                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2043                         struct sk_buff *skb = tx_buf->skb;
2044                         int k, last;
2045
2046                         if (!skb) {
2047                                 j++;
2048                                 continue;
2049                         }
2050
2051                         tx_buf->skb = NULL;
2052
2053                         if (tx_buf->is_push) {
2054                                 dev_kfree_skb(skb);
2055                                 j += 2;
2056                                 continue;
2057                         }
2058
2059                         dma_unmap_single(&pdev->dev,
2060                                          dma_unmap_addr(tx_buf, mapping),
2061                                          skb_headlen(skb),
2062                                          PCI_DMA_TODEVICE);
2063
2064                         last = tx_buf->nr_frags;
2065                         j += 2;
2066                         for (k = 0; k < last; k++, j++) {
2067                                 int ring_idx = j & bp->tx_ring_mask;
2068                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2069
2070                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2071                                 dma_unmap_page(
2072                                         &pdev->dev,
2073                                         dma_unmap_addr(tx_buf, mapping),
2074                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
2075                         }
2076                         dev_kfree_skb(skb);
2077                 }
2078                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2079         }
2080 }
2081
2082 static void bnxt_free_rx_skbs(struct bnxt *bp)
2083 {
2084         int i, max_idx, max_agg_idx;
2085         struct pci_dev *pdev = bp->pdev;
2086
2087         if (!bp->rx_ring)
2088                 return;
2089
2090         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2091         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2092         for (i = 0; i < bp->rx_nr_rings; i++) {
2093                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2094                 int j;
2095
2096                 if (rxr->rx_tpa) {
2097                         for (j = 0; j < MAX_TPA; j++) {
2098                                 struct bnxt_tpa_info *tpa_info =
2099                                                         &rxr->rx_tpa[j];
2100                                 u8 *data = tpa_info->data;
2101
2102                                 if (!data)
2103                                         continue;
2104
2105                                 dma_unmap_single_attrs(&pdev->dev,
2106                                                        tpa_info->mapping,
2107                                                        bp->rx_buf_use_size,
2108                                                        bp->rx_dir,
2109                                                        DMA_ATTR_WEAK_ORDERING);
2110
2111                                 tpa_info->data = NULL;
2112
2113                                 kfree(data);
2114                         }
2115                 }
2116
2117                 for (j = 0; j < max_idx; j++) {
2118                         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
2119                         dma_addr_t mapping = rx_buf->mapping;
2120                         void *data = rx_buf->data;
2121
2122                         if (!data)
2123                                 continue;
2124
2125                         rx_buf->data = NULL;
2126
2127                         if (BNXT_RX_PAGE_MODE(bp)) {
2128                                 mapping -= bp->rx_dma_offset;
2129                                 dma_unmap_page_attrs(&pdev->dev, mapping,
2130                                                      PAGE_SIZE, bp->rx_dir,
2131                                                      DMA_ATTR_WEAK_ORDERING);
2132                                 __free_page(data);
2133                         } else {
2134                                 dma_unmap_single_attrs(&pdev->dev, mapping,
2135                                                        bp->rx_buf_use_size,
2136                                                        bp->rx_dir,
2137                                                        DMA_ATTR_WEAK_ORDERING);
2138                                 kfree(data);
2139                         }
2140                 }
2141
2142                 for (j = 0; j < max_agg_idx; j++) {
2143                         struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2144                                 &rxr->rx_agg_ring[j];
2145                         struct page *page = rx_agg_buf->page;
2146
2147                         if (!page)
2148                                 continue;
2149
2150                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2151                                              BNXT_RX_PAGE_SIZE,
2152                                              PCI_DMA_FROMDEVICE,
2153                                              DMA_ATTR_WEAK_ORDERING);
2154
2155                         rx_agg_buf->page = NULL;
2156                         __clear_bit(j, rxr->rx_agg_bmap);
2157
2158                         __free_page(page);
2159                 }
2160                 if (rxr->rx_page) {
2161                         __free_page(rxr->rx_page);
2162                         rxr->rx_page = NULL;
2163                 }
2164         }
2165 }
2166
2167 static void bnxt_free_skbs(struct bnxt *bp)
2168 {
2169         bnxt_free_tx_skbs(bp);
2170         bnxt_free_rx_skbs(bp);
2171 }
2172
2173 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
2174 {
2175         struct pci_dev *pdev = bp->pdev;
2176         int i;
2177
2178         for (i = 0; i < ring->nr_pages; i++) {
2179                 if (!ring->pg_arr[i])
2180                         continue;
2181
2182                 dma_free_coherent(&pdev->dev, ring->page_size,
2183                                   ring->pg_arr[i], ring->dma_arr[i]);
2184
2185                 ring->pg_arr[i] = NULL;
2186         }
2187         if (ring->pg_tbl) {
2188                 dma_free_coherent(&pdev->dev, ring->nr_pages * 8,
2189                                   ring->pg_tbl, ring->pg_tbl_map);
2190                 ring->pg_tbl = NULL;
2191         }
2192         if (ring->vmem_size && *ring->vmem) {
2193                 vfree(*ring->vmem);
2194                 *ring->vmem = NULL;
2195         }
2196 }
2197
2198 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
2199 {
2200         int i;
2201         struct pci_dev *pdev = bp->pdev;
2202
2203         if (ring->nr_pages > 1) {
2204                 ring->pg_tbl = dma_alloc_coherent(&pdev->dev,
2205                                                   ring->nr_pages * 8,
2206                                                   &ring->pg_tbl_map,
2207                                                   GFP_KERNEL);
2208                 if (!ring->pg_tbl)
2209                         return -ENOMEM;
2210         }
2211
2212         for (i = 0; i < ring->nr_pages; i++) {
2213                 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2214                                                      ring->page_size,
2215                                                      &ring->dma_arr[i],
2216                                                      GFP_KERNEL);
2217                 if (!ring->pg_arr[i])
2218                         return -ENOMEM;
2219
2220                 if (ring->nr_pages > 1)
2221                         ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]);
2222         }
2223
2224         if (ring->vmem_size) {
2225                 *ring->vmem = vzalloc(ring->vmem_size);
2226                 if (!(*ring->vmem))
2227                         return -ENOMEM;
2228         }
2229         return 0;
2230 }
2231
2232 static void bnxt_free_rx_rings(struct bnxt *bp)
2233 {
2234         int i;
2235
2236         if (!bp->rx_ring)
2237                 return;
2238
2239         for (i = 0; i < bp->rx_nr_rings; i++) {
2240                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2241                 struct bnxt_ring_struct *ring;
2242
2243                 if (rxr->xdp_prog)
2244                         bpf_prog_put(rxr->xdp_prog);
2245
2246                 kfree(rxr->rx_tpa);
2247                 rxr->rx_tpa = NULL;
2248
2249                 kfree(rxr->rx_agg_bmap);
2250                 rxr->rx_agg_bmap = NULL;
2251
2252                 ring = &rxr->rx_ring_struct;
2253                 bnxt_free_ring(bp, ring);
2254
2255                 ring = &rxr->rx_agg_ring_struct;
2256                 bnxt_free_ring(bp, ring);
2257         }
2258 }
2259
2260 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2261 {
2262         int i, rc, agg_rings = 0, tpa_rings = 0;
2263
2264         if (!bp->rx_ring)
2265                 return -ENOMEM;
2266
2267         if (bp->flags & BNXT_FLAG_AGG_RINGS)
2268                 agg_rings = 1;
2269
2270         if (bp->flags & BNXT_FLAG_TPA)
2271                 tpa_rings = 1;
2272
2273         for (i = 0; i < bp->rx_nr_rings; i++) {
2274                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2275                 struct bnxt_ring_struct *ring;
2276
2277                 ring = &rxr->rx_ring_struct;
2278
2279                 rc = bnxt_alloc_ring(bp, ring);
2280                 if (rc)
2281                         return rc;
2282
2283                 if (agg_rings) {
2284                         u16 mem_size;
2285
2286                         ring = &rxr->rx_agg_ring_struct;
2287                         rc = bnxt_alloc_ring(bp, ring);
2288                         if (rc)
2289                                 return rc;
2290
2291                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2292                         mem_size = rxr->rx_agg_bmap_size / 8;
2293                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2294                         if (!rxr->rx_agg_bmap)
2295                                 return -ENOMEM;
2296
2297                         if (tpa_rings) {
2298                                 rxr->rx_tpa = kcalloc(MAX_TPA,
2299                                                 sizeof(struct bnxt_tpa_info),
2300                                                 GFP_KERNEL);
2301                                 if (!rxr->rx_tpa)
2302                                         return -ENOMEM;
2303                         }
2304                 }
2305         }
2306         return 0;
2307 }
2308
2309 static void bnxt_free_tx_rings(struct bnxt *bp)
2310 {
2311         int i;
2312         struct pci_dev *pdev = bp->pdev;
2313
2314         if (!bp->tx_ring)
2315                 return;
2316
2317         for (i = 0; i < bp->tx_nr_rings; i++) {
2318                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2319                 struct bnxt_ring_struct *ring;
2320
2321                 if (txr->tx_push) {
2322                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
2323                                           txr->tx_push, txr->tx_push_mapping);
2324                         txr->tx_push = NULL;
2325                 }
2326
2327                 ring = &txr->tx_ring_struct;
2328
2329                 bnxt_free_ring(bp, ring);
2330         }
2331 }
2332
2333 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2334 {
2335         int i, j, rc;
2336         struct pci_dev *pdev = bp->pdev;
2337
2338         bp->tx_push_size = 0;
2339         if (bp->tx_push_thresh) {
2340                 int push_size;
2341
2342                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2343                                         bp->tx_push_thresh);
2344
2345                 if (push_size > 256) {
2346                         push_size = 0;
2347                         bp->tx_push_thresh = 0;
2348                 }
2349
2350                 bp->tx_push_size = push_size;
2351         }
2352
2353         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2354                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2355                 struct bnxt_ring_struct *ring;
2356
2357                 ring = &txr->tx_ring_struct;
2358
2359                 rc = bnxt_alloc_ring(bp, ring);
2360                 if (rc)
2361                         return rc;
2362
2363                 if (bp->tx_push_size) {
2364                         dma_addr_t mapping;
2365
2366                         /* One pre-allocated DMA buffer to backup
2367                          * TX push operation
2368                          */
2369                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
2370                                                 bp->tx_push_size,
2371                                                 &txr->tx_push_mapping,
2372                                                 GFP_KERNEL);
2373
2374                         if (!txr->tx_push)
2375                                 return -ENOMEM;
2376
2377                         mapping = txr->tx_push_mapping +
2378                                 sizeof(struct tx_push_bd);
2379                         txr->data_mapping = cpu_to_le64(mapping);
2380
2381                         memset(txr->tx_push, 0, sizeof(struct tx_push_bd));
2382                 }
2383                 ring->queue_id = bp->q_info[j].queue_id;
2384                 if (i < bp->tx_nr_rings_xdp)
2385                         continue;
2386                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2387                         j++;
2388         }
2389         return 0;
2390 }
2391
2392 static void bnxt_free_cp_rings(struct bnxt *bp)
2393 {
2394         int i;
2395
2396         if (!bp->bnapi)
2397                 return;
2398
2399         for (i = 0; i < bp->cp_nr_rings; i++) {
2400                 struct bnxt_napi *bnapi = bp->bnapi[i];
2401                 struct bnxt_cp_ring_info *cpr;
2402                 struct bnxt_ring_struct *ring;
2403
2404                 if (!bnapi)
2405                         continue;
2406
2407                 cpr = &bnapi->cp_ring;
2408                 ring = &cpr->cp_ring_struct;
2409
2410                 bnxt_free_ring(bp, ring);
2411         }
2412 }
2413
2414 static int bnxt_alloc_cp_rings(struct bnxt *bp)
2415 {
2416         int i, rc;
2417
2418         for (i = 0; i < bp->cp_nr_rings; i++) {
2419                 struct bnxt_napi *bnapi = bp->bnapi[i];
2420                 struct bnxt_cp_ring_info *cpr;
2421                 struct bnxt_ring_struct *ring;
2422
2423                 if (!bnapi)
2424                         continue;
2425
2426                 cpr = &bnapi->cp_ring;
2427                 ring = &cpr->cp_ring_struct;
2428
2429                 rc = bnxt_alloc_ring(bp, ring);
2430                 if (rc)
2431                         return rc;
2432         }
2433         return 0;
2434 }
2435
2436 static void bnxt_init_ring_struct(struct bnxt *bp)
2437 {
2438         int i;
2439
2440         for (i = 0; i < bp->cp_nr_rings; i++) {
2441                 struct bnxt_napi *bnapi = bp->bnapi[i];
2442                 struct bnxt_cp_ring_info *cpr;
2443                 struct bnxt_rx_ring_info *rxr;
2444                 struct bnxt_tx_ring_info *txr;
2445                 struct bnxt_ring_struct *ring;
2446
2447                 if (!bnapi)
2448                         continue;
2449
2450                 cpr = &bnapi->cp_ring;
2451                 ring = &cpr->cp_ring_struct;
2452                 ring->nr_pages = bp->cp_nr_pages;
2453                 ring->page_size = HW_CMPD_RING_SIZE;
2454                 ring->pg_arr = (void **)cpr->cp_desc_ring;
2455                 ring->dma_arr = cpr->cp_desc_mapping;
2456                 ring->vmem_size = 0;
2457
2458                 rxr = bnapi->rx_ring;
2459                 if (!rxr)
2460                         goto skip_rx;
2461
2462                 ring = &rxr->rx_ring_struct;
2463                 ring->nr_pages = bp->rx_nr_pages;
2464                 ring->page_size = HW_RXBD_RING_SIZE;
2465                 ring->pg_arr = (void **)rxr->rx_desc_ring;
2466                 ring->dma_arr = rxr->rx_desc_mapping;
2467                 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
2468                 ring->vmem = (void **)&rxr->rx_buf_ring;
2469
2470                 ring = &rxr->rx_agg_ring_struct;
2471                 ring->nr_pages = bp->rx_agg_nr_pages;
2472                 ring->page_size = HW_RXBD_RING_SIZE;
2473                 ring->pg_arr = (void **)rxr->rx_agg_desc_ring;
2474                 ring->dma_arr = rxr->rx_agg_desc_mapping;
2475                 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
2476                 ring->vmem = (void **)&rxr->rx_agg_ring;
2477
2478 skip_rx:
2479                 txr = bnapi->tx_ring;
2480                 if (!txr)
2481                         continue;
2482
2483                 ring = &txr->tx_ring_struct;
2484                 ring->nr_pages = bp->tx_nr_pages;
2485                 ring->page_size = HW_RXBD_RING_SIZE;
2486                 ring->pg_arr = (void **)txr->tx_desc_ring;
2487                 ring->dma_arr = txr->tx_desc_mapping;
2488                 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
2489                 ring->vmem = (void **)&txr->tx_buf_ring;
2490         }
2491 }
2492
2493 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
2494 {
2495         int i;
2496         u32 prod;
2497         struct rx_bd **rx_buf_ring;
2498
2499         rx_buf_ring = (struct rx_bd **)ring->pg_arr;
2500         for (i = 0, prod = 0; i < ring->nr_pages; i++) {
2501                 int j;
2502                 struct rx_bd *rxbd;
2503
2504                 rxbd = rx_buf_ring[i];
2505                 if (!rxbd)
2506                         continue;
2507
2508                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
2509                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
2510                         rxbd->rx_bd_opaque = prod;
2511                 }
2512         }
2513 }
2514
2515 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
2516 {
2517         struct net_device *dev = bp->dev;
2518         struct bnxt_rx_ring_info *rxr;
2519         struct bnxt_ring_struct *ring;
2520         u32 prod, type;
2521         int i;
2522
2523         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
2524                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
2525
2526         if (NET_IP_ALIGN == 2)
2527                 type |= RX_BD_FLAGS_SOP;
2528
2529         rxr = &bp->rx_ring[ring_nr];
2530         ring = &rxr->rx_ring_struct;
2531         bnxt_init_rxbd_pages(ring, type);
2532
2533         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
2534                 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1);
2535                 if (IS_ERR(rxr->xdp_prog)) {
2536                         int rc = PTR_ERR(rxr->xdp_prog);
2537
2538                         rxr->xdp_prog = NULL;
2539                         return rc;
2540                 }
2541         }
2542         prod = rxr->rx_prod;
2543         for (i = 0; i < bp->rx_ring_size; i++) {
2544                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
2545                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
2546                                     ring_nr, i, bp->rx_ring_size);
2547                         break;
2548                 }
2549                 prod = NEXT_RX(prod);
2550         }
2551         rxr->rx_prod = prod;
2552         ring->fw_ring_id = INVALID_HW_RING_ID;
2553
2554         ring = &rxr->rx_agg_ring_struct;
2555         ring->fw_ring_id = INVALID_HW_RING_ID;
2556
2557         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
2558                 return 0;
2559
2560         type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
2561                 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
2562
2563         bnxt_init_rxbd_pages(ring, type);
2564
2565         prod = rxr->rx_agg_prod;
2566         for (i = 0; i < bp->rx_agg_ring_size; i++) {
2567                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
2568                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
2569                                     ring_nr, i, bp->rx_ring_size);
2570                         break;
2571                 }
2572                 prod = NEXT_RX_AGG(prod);
2573         }
2574         rxr->rx_agg_prod = prod;
2575
2576         if (bp->flags & BNXT_FLAG_TPA) {
2577                 if (rxr->rx_tpa) {
2578                         u8 *data;
2579                         dma_addr_t mapping;
2580
2581                         for (i = 0; i < MAX_TPA; i++) {
2582                                 data = __bnxt_alloc_rx_data(bp, &mapping,
2583                                                             GFP_KERNEL);
2584                                 if (!data)
2585                                         return -ENOMEM;
2586
2587                                 rxr->rx_tpa[i].data = data;
2588                                 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
2589                                 rxr->rx_tpa[i].mapping = mapping;
2590                         }
2591                 } else {
2592                         netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
2593                         return -ENOMEM;
2594                 }
2595         }
2596
2597         return 0;
2598 }
2599
2600 static void bnxt_init_cp_rings(struct bnxt *bp)
2601 {
2602         int i;
2603
2604         for (i = 0; i < bp->cp_nr_rings; i++) {
2605                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
2606                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
2607
2608                 ring->fw_ring_id = INVALID_HW_RING_ID;
2609         }
2610 }
2611
2612 static int bnxt_init_rx_rings(struct bnxt *bp)
2613 {
2614         int i, rc = 0;
2615
2616         if (BNXT_RX_PAGE_MODE(bp)) {
2617                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
2618                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
2619         } else {
2620                 bp->rx_offset = BNXT_RX_OFFSET;
2621                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
2622         }
2623
2624         for (i = 0; i < bp->rx_nr_rings; i++) {
2625                 rc = bnxt_init_one_rx_ring(bp, i);
2626                 if (rc)
2627                         break;
2628         }
2629
2630         return rc;
2631 }
2632
2633 static int bnxt_init_tx_rings(struct bnxt *bp)
2634 {
2635         u16 i;
2636
2637         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
2638                                    MAX_SKB_FRAGS + 1);
2639
2640         for (i = 0; i < bp->tx_nr_rings; i++) {
2641                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2642                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
2643
2644                 ring->fw_ring_id = INVALID_HW_RING_ID;
2645         }
2646
2647         return 0;
2648 }
2649
2650 static void bnxt_free_ring_grps(struct bnxt *bp)
2651 {
2652         kfree(bp->grp_info);
2653         bp->grp_info = NULL;
2654 }
2655
2656 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
2657 {
2658         int i;
2659
2660         if (irq_re_init) {
2661                 bp->grp_info = kcalloc(bp->cp_nr_rings,
2662                                        sizeof(struct bnxt_ring_grp_info),
2663                                        GFP_KERNEL);
2664                 if (!bp->grp_info)
2665                         return -ENOMEM;
2666         }
2667         for (i = 0; i < bp->cp_nr_rings; i++) {
2668                 if (irq_re_init)
2669                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
2670                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
2671                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
2672                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
2673                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
2674         }
2675         return 0;
2676 }
2677
2678 static void bnxt_free_vnics(struct bnxt *bp)
2679 {
2680         kfree(bp->vnic_info);
2681         bp->vnic_info = NULL;
2682         bp->nr_vnics = 0;
2683 }
2684
2685 static int bnxt_alloc_vnics(struct bnxt *bp)
2686 {
2687         int num_vnics = 1;
2688
2689 #ifdef CONFIG_RFS_ACCEL
2690         if (bp->flags & BNXT_FLAG_RFS)
2691                 num_vnics += bp->rx_nr_rings;
2692 #endif
2693
2694         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
2695                 num_vnics++;
2696
2697         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
2698                                 GFP_KERNEL);
2699         if (!bp->vnic_info)
2700                 return -ENOMEM;
2701
2702         bp->nr_vnics = num_vnics;
2703         return 0;
2704 }
2705
2706 static void bnxt_init_vnics(struct bnxt *bp)
2707 {
2708         int i;
2709
2710         for (i = 0; i < bp->nr_vnics; i++) {
2711                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
2712
2713                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
2714                 vnic->fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID;
2715                 vnic->fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID;
2716                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
2717
2718                 if (bp->vnic_info[i].rss_hash_key) {
2719                         if (i == 0)
2720                                 prandom_bytes(vnic->rss_hash_key,
2721                                               HW_HASH_KEY_SIZE);
2722                         else
2723                                 memcpy(vnic->rss_hash_key,
2724                                        bp->vnic_info[0].rss_hash_key,
2725                                        HW_HASH_KEY_SIZE);
2726                 }
2727         }
2728 }
2729
2730 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
2731 {
2732         int pages;
2733
2734         pages = ring_size / desc_per_pg;
2735
2736         if (!pages)
2737                 return 1;
2738
2739         pages++;
2740
2741         while (pages & (pages - 1))
2742                 pages++;
2743
2744         return pages;
2745 }
2746
2747 void bnxt_set_tpa_flags(struct bnxt *bp)
2748 {
2749         bp->flags &= ~BNXT_FLAG_TPA;
2750         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
2751                 return;
2752         if (bp->dev->features & NETIF_F_LRO)
2753                 bp->flags |= BNXT_FLAG_LRO;
2754         if (bp->dev->features & NETIF_F_GRO)
2755                 bp->flags |= BNXT_FLAG_GRO;
2756 }
2757
2758 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
2759  * be set on entry.
2760  */
2761 void bnxt_set_ring_params(struct bnxt *bp)
2762 {
2763         u32 ring_size, rx_size, rx_space;
2764         u32 agg_factor = 0, agg_ring_size = 0;
2765
2766         /* 8 for CRC and VLAN */
2767         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
2768
2769         rx_space = rx_size + NET_SKB_PAD +
2770                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2771
2772         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
2773         ring_size = bp->rx_ring_size;
2774         bp->rx_agg_ring_size = 0;
2775         bp->rx_agg_nr_pages = 0;
2776
2777         if (bp->flags & BNXT_FLAG_TPA)
2778                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
2779
2780         bp->flags &= ~BNXT_FLAG_JUMBO;
2781         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
2782                 u32 jumbo_factor;
2783
2784                 bp->flags |= BNXT_FLAG_JUMBO;
2785                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
2786                 if (jumbo_factor > agg_factor)
2787                         agg_factor = jumbo_factor;
2788         }
2789         agg_ring_size = ring_size * agg_factor;
2790
2791         if (agg_ring_size) {
2792                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
2793                                                         RX_DESC_CNT);
2794                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
2795                         u32 tmp = agg_ring_size;
2796
2797                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
2798                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
2799                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
2800                                     tmp, agg_ring_size);
2801                 }
2802                 bp->rx_agg_ring_size = agg_ring_size;
2803                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
2804                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
2805                 rx_space = rx_size + NET_SKB_PAD +
2806                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2807         }
2808
2809         bp->rx_buf_use_size = rx_size;
2810         bp->rx_buf_size = rx_space;
2811
2812         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
2813         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
2814
2815         ring_size = bp->tx_ring_size;
2816         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
2817         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
2818
2819         ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
2820         bp->cp_ring_size = ring_size;
2821
2822         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
2823         if (bp->cp_nr_pages > MAX_CP_PAGES) {
2824                 bp->cp_nr_pages = MAX_CP_PAGES;
2825                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
2826                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
2827                             ring_size, bp->cp_ring_size);
2828         }
2829         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
2830         bp->cp_ring_mask = bp->cp_bit - 1;
2831 }
2832
2833 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
2834 {
2835         if (page_mode) {
2836                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
2837                         return -EOPNOTSUPP;
2838                 bp->dev->max_mtu =
2839                         min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
2840                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
2841                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
2842                 bp->dev->hw_features &= ~NETIF_F_LRO;
2843                 bp->dev->features &= ~NETIF_F_LRO;
2844                 bp->rx_dir = DMA_BIDIRECTIONAL;
2845                 bp->rx_skb_func = bnxt_rx_page_skb;
2846         } else {
2847                 bp->dev->max_mtu = bp->max_mtu;
2848                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
2849                 bp->rx_dir = DMA_FROM_DEVICE;
2850                 bp->rx_skb_func = bnxt_rx_skb;
2851         }
2852         return 0;
2853 }
2854
2855 static void bnxt_free_vnic_attributes(struct bnxt *bp)
2856 {
2857         int i;
2858         struct bnxt_vnic_info *vnic;
2859         struct pci_dev *pdev = bp->pdev;
2860
2861         if (!bp->vnic_info)
2862                 return;
2863
2864         for (i = 0; i < bp->nr_vnics; i++) {
2865                 vnic = &bp->vnic_info[i];
2866
2867                 kfree(vnic->fw_grp_ids);
2868                 vnic->fw_grp_ids = NULL;
2869
2870                 kfree(vnic->uc_list);
2871                 vnic->uc_list = NULL;
2872
2873                 if (vnic->mc_list) {
2874                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
2875                                           vnic->mc_list, vnic->mc_list_mapping);
2876                         vnic->mc_list = NULL;
2877                 }
2878
2879                 if (vnic->rss_table) {
2880                         dma_free_coherent(&pdev->dev, PAGE_SIZE,
2881                                           vnic->rss_table,
2882                                           vnic->rss_table_dma_addr);
2883                         vnic->rss_table = NULL;
2884                 }
2885
2886                 vnic->rss_hash_key = NULL;
2887                 vnic->flags = 0;
2888         }
2889 }
2890
2891 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
2892 {
2893         int i, rc = 0, size;
2894         struct bnxt_vnic_info *vnic;
2895         struct pci_dev *pdev = bp->pdev;
2896         int max_rings;
2897
2898         for (i = 0; i < bp->nr_vnics; i++) {
2899                 vnic = &bp->vnic_info[i];
2900
2901                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
2902                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
2903
2904                         if (mem_size > 0) {
2905                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
2906                                 if (!vnic->uc_list) {
2907                                         rc = -ENOMEM;
2908                                         goto out;
2909                                 }
2910                         }
2911                 }
2912
2913                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
2914                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
2915                         vnic->mc_list =
2916                                 dma_alloc_coherent(&pdev->dev,
2917                                                    vnic->mc_list_size,
2918                                                    &vnic->mc_list_mapping,
2919                                                    GFP_KERNEL);
2920                         if (!vnic->mc_list) {
2921                                 rc = -ENOMEM;
2922                                 goto out;
2923                         }
2924                 }
2925
2926                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
2927                         max_rings = bp->rx_nr_rings;
2928                 else
2929                         max_rings = 1;
2930
2931                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
2932                 if (!vnic->fw_grp_ids) {
2933                         rc = -ENOMEM;
2934                         goto out;
2935                 }
2936
2937                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
2938                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
2939                         continue;
2940
2941                 /* Allocate rss table and hash key */
2942                 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2943                                                      &vnic->rss_table_dma_addr,
2944                                                      GFP_KERNEL);
2945                 if (!vnic->rss_table) {
2946                         rc = -ENOMEM;
2947                         goto out;
2948                 }
2949
2950                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
2951
2952                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
2953                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
2954         }
2955         return 0;
2956
2957 out:
2958         return rc;
2959 }
2960
2961 static void bnxt_free_hwrm_resources(struct bnxt *bp)
2962 {
2963         struct pci_dev *pdev = bp->pdev;
2964
2965         dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
2966                           bp->hwrm_cmd_resp_dma_addr);
2967
2968         bp->hwrm_cmd_resp_addr = NULL;
2969         if (bp->hwrm_dbg_resp_addr) {
2970                 dma_free_coherent(&pdev->dev, HWRM_DBG_REG_BUF_SIZE,
2971                                   bp->hwrm_dbg_resp_addr,
2972                                   bp->hwrm_dbg_resp_dma_addr);
2973
2974                 bp->hwrm_dbg_resp_addr = NULL;
2975         }
2976 }
2977
2978 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
2979 {
2980         struct pci_dev *pdev = bp->pdev;
2981
2982         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2983                                                    &bp->hwrm_cmd_resp_dma_addr,
2984                                                    GFP_KERNEL);
2985         if (!bp->hwrm_cmd_resp_addr)
2986                 return -ENOMEM;
2987         bp->hwrm_dbg_resp_addr = dma_alloc_coherent(&pdev->dev,
2988                                                     HWRM_DBG_REG_BUF_SIZE,
2989                                                     &bp->hwrm_dbg_resp_dma_addr,
2990                                                     GFP_KERNEL);
2991         if (!bp->hwrm_dbg_resp_addr)
2992                 netdev_warn(bp->dev, "fail to alloc debug register dma mem\n");
2993
2994         return 0;
2995 }
2996
2997 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
2998 {
2999         if (bp->hwrm_short_cmd_req_addr) {
3000                 struct pci_dev *pdev = bp->pdev;
3001
3002                 dma_free_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN,
3003                                   bp->hwrm_short_cmd_req_addr,
3004                                   bp->hwrm_short_cmd_req_dma_addr);
3005                 bp->hwrm_short_cmd_req_addr = NULL;
3006         }
3007 }
3008
3009 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3010 {
3011         struct pci_dev *pdev = bp->pdev;
3012
3013         bp->hwrm_short_cmd_req_addr =
3014                 dma_alloc_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN,
3015                                    &bp->hwrm_short_cmd_req_dma_addr,
3016                                    GFP_KERNEL);
3017         if (!bp->hwrm_short_cmd_req_addr)
3018                 return -ENOMEM;
3019
3020         return 0;
3021 }
3022
3023 static void bnxt_free_stats(struct bnxt *bp)
3024 {
3025         u32 size, i;
3026         struct pci_dev *pdev = bp->pdev;
3027
3028         if (bp->hw_rx_port_stats) {
3029                 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
3030                                   bp->hw_rx_port_stats,
3031                                   bp->hw_rx_port_stats_map);
3032                 bp->hw_rx_port_stats = NULL;
3033                 bp->flags &= ~BNXT_FLAG_PORT_STATS;
3034         }
3035
3036         if (!bp->bnapi)
3037                 return;
3038
3039         size = sizeof(struct ctx_hw_stats);
3040
3041         for (i = 0; i < bp->cp_nr_rings; i++) {
3042                 struct bnxt_napi *bnapi = bp->bnapi[i];
3043                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3044
3045                 if (cpr->hw_stats) {
3046                         dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
3047                                           cpr->hw_stats_map);
3048                         cpr->hw_stats = NULL;
3049                 }
3050         }
3051 }
3052
3053 static int bnxt_alloc_stats(struct bnxt *bp)
3054 {
3055         u32 size, i;
3056         struct pci_dev *pdev = bp->pdev;
3057
3058         size = sizeof(struct ctx_hw_stats);
3059
3060         for (i = 0; i < bp->cp_nr_rings; i++) {
3061                 struct bnxt_napi *bnapi = bp->bnapi[i];
3062                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3063
3064                 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
3065                                                    &cpr->hw_stats_map,
3066                                                    GFP_KERNEL);
3067                 if (!cpr->hw_stats)
3068                         return -ENOMEM;
3069
3070                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3071         }
3072
3073         if (BNXT_PF(bp) && bp->chip_num != CHIP_NUM_58700) {
3074                 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
3075                                          sizeof(struct tx_port_stats) + 1024;
3076
3077                 bp->hw_rx_port_stats =
3078                         dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
3079                                            &bp->hw_rx_port_stats_map,
3080                                            GFP_KERNEL);
3081                 if (!bp->hw_rx_port_stats)
3082                         return -ENOMEM;
3083
3084                 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) +
3085                                        512;
3086                 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
3087                                            sizeof(struct rx_port_stats) + 512;
3088                 bp->flags |= BNXT_FLAG_PORT_STATS;
3089         }
3090         return 0;
3091 }
3092
3093 static void bnxt_clear_ring_indices(struct bnxt *bp)
3094 {
3095         int i;
3096
3097         if (!bp->bnapi)
3098                 return;
3099
3100         for (i = 0; i < bp->cp_nr_rings; i++) {
3101                 struct bnxt_napi *bnapi = bp->bnapi[i];
3102                 struct bnxt_cp_ring_info *cpr;
3103                 struct bnxt_rx_ring_info *rxr;
3104                 struct bnxt_tx_ring_info *txr;
3105
3106                 if (!bnapi)
3107                         continue;
3108
3109                 cpr = &bnapi->cp_ring;
3110                 cpr->cp_raw_cons = 0;
3111
3112                 txr = bnapi->tx_ring;
3113                 if (txr) {
3114                         txr->tx_prod = 0;
3115                         txr->tx_cons = 0;
3116                 }
3117
3118                 rxr = bnapi->rx_ring;
3119                 if (rxr) {
3120                         rxr->rx_prod = 0;
3121                         rxr->rx_agg_prod = 0;
3122                         rxr->rx_sw_agg_prod = 0;
3123                         rxr->rx_next_cons = 0;
3124                 }
3125         }
3126 }
3127
3128 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3129 {
3130 #ifdef CONFIG_RFS_ACCEL
3131         int i;
3132
3133         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
3134          * safe to delete the hash table.
3135          */
3136         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
3137                 struct hlist_head *head;
3138                 struct hlist_node *tmp;
3139                 struct bnxt_ntuple_filter *fltr;
3140
3141                 head = &bp->ntp_fltr_hash_tbl[i];
3142                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
3143                         hlist_del(&fltr->hash);
3144                         kfree(fltr);
3145                 }
3146         }
3147         if (irq_reinit) {
3148                 kfree(bp->ntp_fltr_bmap);
3149                 bp->ntp_fltr_bmap = NULL;
3150         }
3151         bp->ntp_fltr_count = 0;
3152 #endif
3153 }
3154
3155 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
3156 {
3157 #ifdef CONFIG_RFS_ACCEL
3158         int i, rc = 0;
3159
3160         if (!(bp->flags & BNXT_FLAG_RFS))
3161                 return 0;
3162
3163         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
3164                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
3165
3166         bp->ntp_fltr_count = 0;
3167         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
3168                                     sizeof(long),
3169                                     GFP_KERNEL);
3170
3171         if (!bp->ntp_fltr_bmap)
3172                 rc = -ENOMEM;
3173
3174         return rc;
3175 #else
3176         return 0;
3177 #endif
3178 }
3179
3180 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
3181 {
3182         bnxt_free_vnic_attributes(bp);
3183         bnxt_free_tx_rings(bp);
3184         bnxt_free_rx_rings(bp);
3185         bnxt_free_cp_rings(bp);
3186         bnxt_free_ntp_fltrs(bp, irq_re_init);
3187         if (irq_re_init) {
3188                 bnxt_free_stats(bp);
3189                 bnxt_free_ring_grps(bp);
3190                 bnxt_free_vnics(bp);
3191                 kfree(bp->tx_ring_map);
3192                 bp->tx_ring_map = NULL;
3193                 kfree(bp->tx_ring);
3194                 bp->tx_ring = NULL;
3195                 kfree(bp->rx_ring);
3196                 bp->rx_ring = NULL;
3197                 kfree(bp->bnapi);
3198                 bp->bnapi = NULL;
3199         } else {
3200                 bnxt_clear_ring_indices(bp);
3201         }
3202 }
3203
3204 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
3205 {
3206         int i, j, rc, size, arr_size;
3207         void *bnapi;
3208
3209         if (irq_re_init) {
3210                 /* Allocate bnapi mem pointer array and mem block for
3211                  * all queues
3212                  */
3213                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
3214                                 bp->cp_nr_rings);
3215                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
3216                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
3217                 if (!bnapi)
3218                         return -ENOMEM;
3219
3220                 bp->bnapi = bnapi;
3221                 bnapi += arr_size;
3222                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
3223                         bp->bnapi[i] = bnapi;
3224                         bp->bnapi[i]->index = i;
3225                         bp->bnapi[i]->bp = bp;
3226                 }
3227
3228                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
3229                                       sizeof(struct bnxt_rx_ring_info),
3230                                       GFP_KERNEL);
3231                 if (!bp->rx_ring)
3232                         return -ENOMEM;
3233
3234                 for (i = 0; i < bp->rx_nr_rings; i++) {
3235                         bp->rx_ring[i].bnapi = bp->bnapi[i];
3236                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
3237                 }
3238
3239                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
3240                                       sizeof(struct bnxt_tx_ring_info),
3241                                       GFP_KERNEL);
3242                 if (!bp->tx_ring)
3243                         return -ENOMEM;
3244
3245                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
3246                                           GFP_KERNEL);
3247
3248                 if (!bp->tx_ring_map)
3249                         return -ENOMEM;
3250
3251                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
3252                         j = 0;
3253                 else
3254                         j = bp->rx_nr_rings;
3255
3256                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
3257                         bp->tx_ring[i].bnapi = bp->bnapi[j];
3258                         bp->bnapi[j]->tx_ring = &bp->tx_ring[i];
3259                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
3260                         if (i >= bp->tx_nr_rings_xdp) {
3261                                 bp->tx_ring[i].txq_index = i -
3262                                         bp->tx_nr_rings_xdp;
3263                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
3264                         } else {
3265                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
3266                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
3267                         }
3268                 }
3269
3270                 rc = bnxt_alloc_stats(bp);
3271                 if (rc)
3272                         goto alloc_mem_err;
3273
3274                 rc = bnxt_alloc_ntp_fltrs(bp);
3275                 if (rc)
3276                         goto alloc_mem_err;
3277
3278                 rc = bnxt_alloc_vnics(bp);
3279                 if (rc)
3280                         goto alloc_mem_err;
3281         }
3282
3283         bnxt_init_ring_struct(bp);
3284
3285         rc = bnxt_alloc_rx_rings(bp);
3286         if (rc)
3287                 goto alloc_mem_err;
3288
3289         rc = bnxt_alloc_tx_rings(bp);
3290         if (rc)
3291                 goto alloc_mem_err;
3292
3293         rc = bnxt_alloc_cp_rings(bp);
3294         if (rc)
3295                 goto alloc_mem_err;
3296
3297         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
3298                                   BNXT_VNIC_UCAST_FLAG;
3299         rc = bnxt_alloc_vnic_attributes(bp);
3300         if (rc)
3301                 goto alloc_mem_err;
3302         return 0;
3303
3304 alloc_mem_err:
3305         bnxt_free_mem(bp, true);
3306         return rc;
3307 }
3308
3309 static void bnxt_disable_int(struct bnxt *bp)
3310 {
3311         int i;
3312
3313         if (!bp->bnapi)
3314                 return;
3315
3316         for (i = 0; i < bp->cp_nr_rings; i++) {
3317                 struct bnxt_napi *bnapi = bp->bnapi[i];
3318                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3319                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3320
3321                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
3322                         BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
3323         }
3324 }
3325
3326 static void bnxt_disable_int_sync(struct bnxt *bp)
3327 {
3328         int i;
3329
3330         atomic_inc(&bp->intr_sem);
3331
3332         bnxt_disable_int(bp);
3333         for (i = 0; i < bp->cp_nr_rings; i++)
3334                 synchronize_irq(bp->irq_tbl[i].vector);
3335 }
3336
3337 static void bnxt_enable_int(struct bnxt *bp)
3338 {
3339         int i;
3340
3341         atomic_set(&bp->intr_sem, 0);
3342         for (i = 0; i < bp->cp_nr_rings; i++) {
3343                 struct bnxt_napi *bnapi = bp->bnapi[i];
3344                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3345
3346                 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
3347         }
3348 }
3349
3350 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
3351                             u16 cmpl_ring, u16 target_id)
3352 {
3353         struct input *req = request;
3354
3355         req->req_type = cpu_to_le16(req_type);
3356         req->cmpl_ring = cpu_to_le16(cmpl_ring);
3357         req->target_id = cpu_to_le16(target_id);
3358         req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
3359 }
3360
3361 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
3362                                  int timeout, bool silent)
3363 {
3364         int i, intr_process, rc, tmo_count;
3365         struct input *req = msg;
3366         u32 *data = msg;
3367         __le32 *resp_len, *valid;
3368         u16 cp_ring_id, len = 0;
3369         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
3370         u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
3371         struct hwrm_short_input short_input = {0};
3372
3373         req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++);
3374         memset(resp, 0, PAGE_SIZE);
3375         cp_ring_id = le16_to_cpu(req->cmpl_ring);
3376         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
3377
3378         if (bp->flags & BNXT_FLAG_SHORT_CMD) {
3379                 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
3380
3381                 memcpy(short_cmd_req, req, msg_len);
3382                 memset(short_cmd_req + msg_len, 0, BNXT_HWRM_MAX_REQ_LEN -
3383                                                    msg_len);
3384
3385                 short_input.req_type = req->req_type;
3386                 short_input.signature =
3387                                 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
3388                 short_input.size = cpu_to_le16(msg_len);
3389                 short_input.req_addr =
3390                         cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
3391
3392                 data = (u32 *)&short_input;
3393                 msg_len = sizeof(short_input);
3394
3395                 /* Sync memory write before updating doorbell */
3396                 wmb();
3397
3398                 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
3399         }
3400
3401         /* Write request msg to hwrm channel */
3402         __iowrite32_copy(bp->bar0, data, msg_len / 4);
3403
3404         for (i = msg_len; i < max_req_len; i += 4)
3405                 writel(0, bp->bar0 + i);
3406
3407         /* currently supports only one outstanding message */
3408         if (intr_process)
3409                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
3410
3411         /* Ring channel doorbell */
3412         writel(1, bp->bar0 + 0x100);
3413
3414         if (!timeout)
3415                 timeout = DFLT_HWRM_CMD_TIMEOUT;
3416
3417         i = 0;
3418         tmo_count = timeout * 40;
3419         if (intr_process) {
3420                 /* Wait until hwrm response cmpl interrupt is processed */
3421                 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID &&
3422                        i++ < tmo_count) {
3423                         usleep_range(25, 40);
3424                 }
3425
3426                 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) {
3427                         netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
3428                                    le16_to_cpu(req->req_type));
3429                         return -1;
3430                 }
3431         } else {
3432                 /* Check if response len is updated */
3433                 resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET;
3434                 for (i = 0; i < tmo_count; i++) {
3435                         len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
3436                               HWRM_RESP_LEN_SFT;
3437                         if (len)
3438                                 break;
3439                         usleep_range(25, 40);
3440                 }
3441
3442                 if (i >= tmo_count) {
3443                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
3444                                    timeout, le16_to_cpu(req->req_type),
3445                                    le16_to_cpu(req->seq_id), len);
3446                         return -1;
3447                 }
3448
3449                 /* Last word of resp contains valid bit */
3450                 valid = bp->hwrm_cmd_resp_addr + len - 4;
3451                 for (i = 0; i < 5; i++) {
3452                         if (le32_to_cpu(*valid) & HWRM_RESP_VALID_MASK)
3453                                 break;
3454                         udelay(1);
3455                 }
3456
3457                 if (i >= 5) {
3458                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
3459                                    timeout, le16_to_cpu(req->req_type),
3460                                    le16_to_cpu(req->seq_id), len, *valid);
3461                         return -1;
3462                 }
3463         }
3464
3465         rc = le16_to_cpu(resp->error_code);
3466         if (rc && !silent)
3467                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
3468                            le16_to_cpu(resp->req_type),
3469                            le16_to_cpu(resp->seq_id), rc);
3470         return rc;
3471 }
3472
3473 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
3474 {
3475         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
3476 }
3477
3478 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
3479                               int timeout)
3480 {
3481         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
3482 }
3483
3484 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
3485 {
3486         int rc;
3487
3488         mutex_lock(&bp->hwrm_cmd_lock);
3489         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
3490         mutex_unlock(&bp->hwrm_cmd_lock);
3491         return rc;
3492 }
3493
3494 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
3495                              int timeout)
3496 {
3497         int rc;
3498
3499         mutex_lock(&bp->hwrm_cmd_lock);
3500         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
3501         mutex_unlock(&bp->hwrm_cmd_lock);
3502         return rc;
3503 }
3504
3505 int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap,
3506                                      int bmap_size)
3507 {
3508         struct hwrm_func_drv_rgtr_input req = {0};
3509         DECLARE_BITMAP(async_events_bmap, 256);
3510         u32 *events = (u32 *)async_events_bmap;
3511         int i;
3512
3513         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
3514
3515         req.enables =
3516                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
3517
3518         memset(async_events_bmap, 0, sizeof(async_events_bmap));
3519         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
3520                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
3521
3522         if (bmap && bmap_size) {
3523                 for (i = 0; i < bmap_size; i++) {
3524                         if (test_bit(i, bmap))
3525                                 __set_bit(i, async_events_bmap);
3526                 }
3527         }
3528
3529         for (i = 0; i < 8; i++)
3530                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
3531
3532         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3533 }
3534
3535 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
3536 {
3537         struct hwrm_func_drv_rgtr_input req = {0};
3538
3539         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
3540
3541         req.enables =
3542                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
3543                             FUNC_DRV_RGTR_REQ_ENABLES_VER);
3544
3545         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
3546         req.ver_maj = DRV_VER_MAJ;
3547         req.ver_min = DRV_VER_MIN;
3548         req.ver_upd = DRV_VER_UPD;
3549
3550         if (BNXT_PF(bp)) {
3551                 u32 data[8];
3552                 int i;
3553
3554                 memset(data, 0, sizeof(data));
3555                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
3556                         u16 cmd = bnxt_vf_req_snif[i];
3557                         unsigned int bit, idx;
3558
3559                         idx = cmd / 32;
3560                         bit = cmd % 32;
3561                         data[idx] |= 1 << bit;
3562                 }
3563
3564                 for (i = 0; i < 8; i++)
3565                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
3566
3567                 req.enables |=
3568                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
3569         }
3570
3571         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3572 }
3573
3574 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
3575 {
3576         struct hwrm_func_drv_unrgtr_input req = {0};
3577
3578         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
3579         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3580 }
3581
3582 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
3583 {
3584         u32 rc = 0;
3585         struct hwrm_tunnel_dst_port_free_input req = {0};
3586
3587         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
3588         req.tunnel_type = tunnel_type;
3589
3590         switch (tunnel_type) {
3591         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
3592                 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
3593                 break;
3594         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
3595                 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
3596                 break;
3597         default:
3598                 break;
3599         }
3600
3601         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3602         if (rc)
3603                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
3604                            rc);
3605         return rc;
3606 }
3607
3608 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
3609                                            u8 tunnel_type)
3610 {
3611         u32 rc = 0;
3612         struct hwrm_tunnel_dst_port_alloc_input req = {0};
3613         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3614
3615         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
3616
3617         req.tunnel_type = tunnel_type;
3618         req.tunnel_dst_port_val = port;
3619
3620         mutex_lock(&bp->hwrm_cmd_lock);
3621         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3622         if (rc) {
3623                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
3624                            rc);
3625                 goto err_out;
3626         }
3627
3628         switch (tunnel_type) {
3629         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
3630                 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
3631                 break;
3632         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
3633                 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
3634                 break;
3635         default:
3636                 break;
3637         }
3638
3639 err_out:
3640         mutex_unlock(&bp->hwrm_cmd_lock);
3641         return rc;
3642 }
3643
3644 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
3645 {
3646         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
3647         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3648
3649         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
3650         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
3651
3652         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
3653         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
3654         req.mask = cpu_to_le32(vnic->rx_mask);
3655         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3656 }
3657
3658 #ifdef CONFIG_RFS_ACCEL
3659 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
3660                                             struct bnxt_ntuple_filter *fltr)
3661 {
3662         struct hwrm_cfa_ntuple_filter_free_input req = {0};
3663
3664         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
3665         req.ntuple_filter_id = fltr->filter_id;
3666         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3667 }
3668
3669 #define BNXT_NTP_FLTR_FLAGS                                     \
3670         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
3671          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
3672          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
3673          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
3674          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
3675          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
3676          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
3677          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
3678          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
3679          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
3680          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
3681          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
3682          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
3683          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
3684
3685 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
3686                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
3687
3688 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
3689                                              struct bnxt_ntuple_filter *fltr)
3690 {
3691         int rc = 0;
3692         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
3693         struct hwrm_cfa_ntuple_filter_alloc_output *resp =
3694                 bp->hwrm_cmd_resp_addr;
3695         struct flow_keys *keys = &fltr->fkeys;
3696         struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1];
3697
3698         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
3699         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
3700
3701         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
3702
3703         req.ethertype = htons(ETH_P_IP);
3704         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
3705         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
3706         req.ip_protocol = keys->basic.ip_proto;
3707
3708         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
3709                 int i;
3710
3711                 req.ethertype = htons(ETH_P_IPV6);
3712                 req.ip_addr_type =
3713                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
3714                 *(struct in6_addr *)&req.src_ipaddr[0] =
3715                         keys->addrs.v6addrs.src;
3716                 *(struct in6_addr *)&req.dst_ipaddr[0] =
3717                         keys->addrs.v6addrs.dst;
3718                 for (i = 0; i < 4; i++) {
3719                         req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
3720                         req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
3721                 }
3722         } else {
3723                 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
3724                 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3725                 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
3726                 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3727         }
3728         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
3729                 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
3730                 req.tunnel_type =
3731                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
3732         }
3733
3734         req.src_port = keys->ports.src;
3735         req.src_port_mask = cpu_to_be16(0xffff);
3736         req.dst_port = keys->ports.dst;
3737         req.dst_port_mask = cpu_to_be16(0xffff);
3738
3739         req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
3740         mutex_lock(&bp->hwrm_cmd_lock);
3741         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3742         if (!rc)
3743                 fltr->filter_id = resp->ntuple_filter_id;
3744         mutex_unlock(&bp->hwrm_cmd_lock);
3745         return rc;
3746 }
3747 #endif
3748
3749 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
3750                                      u8 *mac_addr)
3751 {
3752         u32 rc = 0;
3753         struct hwrm_cfa_l2_filter_alloc_input req = {0};
3754         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3755
3756         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
3757         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
3758         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
3759                 req.flags |=
3760                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
3761         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
3762         req.enables =
3763                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
3764                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
3765                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
3766         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
3767         req.l2_addr_mask[0] = 0xff;
3768         req.l2_addr_mask[1] = 0xff;
3769         req.l2_addr_mask[2] = 0xff;
3770         req.l2_addr_mask[3] = 0xff;
3771         req.l2_addr_mask[4] = 0xff;
3772         req.l2_addr_mask[5] = 0xff;
3773
3774         mutex_lock(&bp->hwrm_cmd_lock);
3775         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3776         if (!rc)
3777                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
3778                                                         resp->l2_filter_id;
3779         mutex_unlock(&bp->hwrm_cmd_lock);
3780         return rc;
3781 }
3782
3783 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
3784 {
3785         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
3786         int rc = 0;
3787
3788         /* Any associated ntuple filters will also be cleared by firmware. */
3789         mutex_lock(&bp->hwrm_cmd_lock);
3790         for (i = 0; i < num_of_vnics; i++) {
3791                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3792
3793                 for (j = 0; j < vnic->uc_filter_count; j++) {
3794                         struct hwrm_cfa_l2_filter_free_input req = {0};
3795
3796                         bnxt_hwrm_cmd_hdr_init(bp, &req,
3797                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
3798
3799                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
3800
3801                         rc = _hwrm_send_message(bp, &req, sizeof(req),
3802                                                 HWRM_CMD_TIMEOUT);
3803                 }
3804                 vnic->uc_filter_count = 0;
3805         }
3806         mutex_unlock(&bp->hwrm_cmd_lock);
3807
3808         return rc;
3809 }
3810
3811 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
3812 {
3813         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3814         struct hwrm_vnic_tpa_cfg_input req = {0};
3815
3816         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
3817
3818         if (tpa_flags) {
3819                 u16 mss = bp->dev->mtu - 40;
3820                 u32 nsegs, n, segs = 0, flags;
3821
3822                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
3823                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
3824                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
3825                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
3826                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
3827                 if (tpa_flags & BNXT_FLAG_GRO)
3828                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
3829
3830                 req.flags = cpu_to_le32(flags);
3831
3832                 req.enables =
3833                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
3834                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
3835                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
3836
3837                 /* Number of segs are log2 units, and first packet is not
3838                  * included as part of this units.
3839                  */
3840                 if (mss <= BNXT_RX_PAGE_SIZE) {
3841                         n = BNXT_RX_PAGE_SIZE / mss;
3842                         nsegs = (MAX_SKB_FRAGS - 1) * n;
3843                 } else {
3844                         n = mss / BNXT_RX_PAGE_SIZE;
3845                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
3846                                 n++;
3847                         nsegs = (MAX_SKB_FRAGS - n) / n;
3848                 }
3849
3850                 segs = ilog2(nsegs);
3851                 req.max_agg_segs = cpu_to_le16(segs);
3852                 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX);
3853
3854                 req.min_agg_len = cpu_to_le32(512);
3855         }
3856         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3857
3858         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3859 }
3860
3861 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
3862 {
3863         u32 i, j, max_rings;
3864         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3865         struct hwrm_vnic_rss_cfg_input req = {0};
3866
3867         if (vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
3868                 return 0;
3869
3870         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
3871         if (set_rss) {
3872                 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
3873                 if (vnic->flags & BNXT_VNIC_RSS_FLAG) {
3874                         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3875                                 max_rings = bp->rx_nr_rings - 1;
3876                         else
3877                                 max_rings = bp->rx_nr_rings;
3878                 } else {
3879                         max_rings = 1;
3880                 }
3881
3882                 /* Fill the RSS indirection table with ring group ids */
3883                 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
3884                         if (j == max_rings)
3885                                 j = 0;
3886                         vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
3887                 }
3888
3889                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
3890                 req.hash_key_tbl_addr =
3891                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
3892         }
3893         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
3894         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3895 }
3896
3897 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
3898 {
3899         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3900         struct hwrm_vnic_plcmodes_cfg_input req = {0};
3901
3902         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
3903         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
3904                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
3905                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
3906         req.enables =
3907                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
3908                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
3909         /* thresholds not implemented in firmware yet */
3910         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
3911         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
3912         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
3913         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3914 }
3915
3916 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
3917                                         u16 ctx_idx)
3918 {
3919         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
3920
3921         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
3922         req.rss_cos_lb_ctx_id =
3923                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
3924
3925         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3926         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
3927 }
3928
3929 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
3930 {
3931         int i, j;
3932
3933         for (i = 0; i < bp->nr_vnics; i++) {
3934                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3935
3936                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
3937                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
3938                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
3939                 }
3940         }
3941         bp->rsscos_nr_ctxs = 0;
3942 }
3943
3944 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
3945 {
3946         int rc;
3947         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
3948         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
3949                                                 bp->hwrm_cmd_resp_addr;
3950
3951         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
3952                                -1);
3953
3954         mutex_lock(&bp->hwrm_cmd_lock);
3955         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3956         if (!rc)
3957                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
3958                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
3959         mutex_unlock(&bp->hwrm_cmd_lock);
3960
3961         return rc;
3962 }
3963
3964 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
3965 {
3966         unsigned int ring = 0, grp_idx;
3967         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3968         struct hwrm_vnic_cfg_input req = {0};
3969         u16 def_vlan = 0;
3970
3971         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
3972
3973         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
3974         /* Only RSS support for now TBD: COS & LB */
3975         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
3976                 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
3977                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
3978                                            VNIC_CFG_REQ_ENABLES_MRU);
3979         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
3980                 req.rss_rule =
3981                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
3982                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
3983                                            VNIC_CFG_REQ_ENABLES_MRU);
3984                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
3985         } else {
3986                 req.rss_rule = cpu_to_le16(0xffff);
3987         }
3988
3989         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
3990             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
3991                 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
3992                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
3993         } else {
3994                 req.cos_rule = cpu_to_le16(0xffff);
3995         }
3996
3997         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3998                 ring = 0;
3999         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
4000                 ring = vnic_id - 1;
4001         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
4002                 ring = bp->rx_nr_rings - 1;
4003
4004         grp_idx = bp->rx_ring[ring].bnapi->index;
4005         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4006         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
4007
4008         req.lb_rule = cpu_to_le16(0xffff);
4009         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
4010                               VLAN_HLEN);
4011
4012 #ifdef CONFIG_BNXT_SRIOV
4013         if (BNXT_VF(bp))
4014                 def_vlan = bp->vf.vlan;
4015 #endif
4016         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
4017                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
4018         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
4019                 req.flags |=
4020                         cpu_to_le32(VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE);
4021
4022         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4023 }
4024
4025 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
4026 {
4027         u32 rc = 0;
4028
4029         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
4030                 struct hwrm_vnic_free_input req = {0};
4031
4032                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
4033                 req.vnic_id =
4034                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
4035
4036                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4037                 if (rc)
4038                         return rc;
4039                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
4040         }
4041         return rc;
4042 }
4043
4044 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
4045 {
4046         u16 i;
4047
4048         for (i = 0; i < bp->nr_vnics; i++)
4049                 bnxt_hwrm_vnic_free_one(bp, i);
4050 }
4051
4052 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
4053                                 unsigned int start_rx_ring_idx,
4054                                 unsigned int nr_rings)
4055 {
4056         int rc = 0;
4057         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
4058         struct hwrm_vnic_alloc_input req = {0};
4059         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4060
4061         /* map ring groups to this vnic */
4062         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
4063                 grp_idx = bp->rx_ring[i].bnapi->index;
4064                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
4065                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
4066                                    j, nr_rings);
4067                         break;
4068                 }
4069                 bp->vnic_info[vnic_id].fw_grp_ids[j] =
4070                                         bp->grp_info[grp_idx].fw_grp_id;
4071         }
4072
4073         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID;
4074         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID;
4075         if (vnic_id == 0)
4076                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
4077
4078         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
4079
4080         mutex_lock(&bp->hwrm_cmd_lock);
4081         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4082         if (!rc)
4083                 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id);
4084         mutex_unlock(&bp->hwrm_cmd_lock);
4085         return rc;
4086 }
4087
4088 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
4089 {
4090         struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4091         struct hwrm_vnic_qcaps_input req = {0};
4092         int rc;
4093
4094         if (bp->hwrm_spec_code < 0x10600)
4095                 return 0;
4096
4097         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
4098         mutex_lock(&bp->hwrm_cmd_lock);
4099         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4100         if (!rc) {
4101                 if (resp->flags &
4102                     cpu_to_le32(VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
4103                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
4104         }
4105         mutex_unlock(&bp->hwrm_cmd_lock);
4106         return rc;
4107 }
4108
4109 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
4110 {
4111         u16 i;
4112         u32 rc = 0;
4113
4114         mutex_lock(&bp->hwrm_cmd_lock);
4115         for (i = 0; i < bp->rx_nr_rings; i++) {
4116                 struct hwrm_ring_grp_alloc_input req = {0};
4117                 struct hwrm_ring_grp_alloc_output *resp =
4118                                         bp->hwrm_cmd_resp_addr;
4119                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
4120
4121                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
4122
4123                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
4124                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
4125                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
4126                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
4127
4128                 rc = _hwrm_send_message(bp, &req, sizeof(req),
4129                                         HWRM_CMD_TIMEOUT);
4130                 if (rc)
4131                         break;
4132
4133                 bp->grp_info[grp_idx].fw_grp_id =
4134                         le32_to_cpu(resp->ring_group_id);
4135         }
4136         mutex_unlock(&bp->hwrm_cmd_lock);
4137         return rc;
4138 }
4139
4140 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
4141 {
4142         u16 i;
4143         u32 rc = 0;
4144         struct hwrm_ring_grp_free_input req = {0};
4145
4146         if (!bp->grp_info)
4147                 return 0;
4148
4149         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
4150
4151         mutex_lock(&bp->hwrm_cmd_lock);
4152         for (i = 0; i < bp->cp_nr_rings; i++) {
4153                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
4154                         continue;
4155                 req.ring_group_id =
4156                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
4157
4158                 rc = _hwrm_send_message(bp, &req, sizeof(req),
4159                                         HWRM_CMD_TIMEOUT);
4160                 if (rc)
4161                         break;
4162                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
4163         }
4164         mutex_unlock(&bp->hwrm_cmd_lock);
4165         return rc;
4166 }
4167
4168 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
4169                                     struct bnxt_ring_struct *ring,
4170                                     u32 ring_type, u32 map_index,
4171                                     u32 stats_ctx_id)
4172 {
4173         int rc = 0, err = 0;
4174         struct hwrm_ring_alloc_input req = {0};
4175         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4176         u16 ring_id;
4177
4178         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
4179
4180         req.enables = 0;
4181         if (ring->nr_pages > 1) {
4182                 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map);
4183                 /* Page size is in log2 units */
4184                 req.page_size = BNXT_PAGE_SHIFT;
4185                 req.page_tbl_depth = 1;
4186         } else {
4187                 req.page_tbl_addr =  cpu_to_le64(ring->dma_arr[0]);
4188         }
4189         req.fbo = 0;
4190         /* Association of ring index with doorbell index and MSIX number */
4191         req.logical_id = cpu_to_le16(map_index);
4192
4193         switch (ring_type) {
4194         case HWRM_RING_ALLOC_TX:
4195                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
4196                 /* Association of transmit ring with completion ring */
4197                 req.cmpl_ring_id =
4198                         cpu_to_le16(bp->grp_info[map_index].cp_fw_ring_id);
4199                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
4200                 req.stat_ctx_id = cpu_to_le32(stats_ctx_id);
4201                 req.queue_id = cpu_to_le16(ring->queue_id);
4202                 break;
4203         case HWRM_RING_ALLOC_RX:
4204                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
4205                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
4206                 break;
4207         case HWRM_RING_ALLOC_AGG:
4208                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
4209                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
4210                 break;
4211         case HWRM_RING_ALLOC_CMPL:
4212                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
4213                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
4214                 if (bp->flags & BNXT_FLAG_USING_MSIX)
4215                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
4216                 break;
4217         default:
4218                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
4219                            ring_type);
4220                 return -1;
4221         }
4222
4223         mutex_lock(&bp->hwrm_cmd_lock);
4224         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4225         err = le16_to_cpu(resp->error_code);
4226         ring_id = le16_to_cpu(resp->ring_id);
4227         mutex_unlock(&bp->hwrm_cmd_lock);
4228
4229         if (rc || err) {
4230                 switch (ring_type) {
4231                 case RING_FREE_REQ_RING_TYPE_L2_CMPL:
4232                         netdev_err(bp->dev, "hwrm_ring_alloc cp failed. rc:%x err:%x\n",
4233                                    rc, err);
4234                         return -1;
4235
4236                 case RING_FREE_REQ_RING_TYPE_RX:
4237                         netdev_err(bp->dev, "hwrm_ring_alloc rx failed. rc:%x err:%x\n",
4238                                    rc, err);
4239                         return -1;
4240
4241                 case RING_FREE_REQ_RING_TYPE_TX:
4242                         netdev_err(bp->dev, "hwrm_ring_alloc tx failed. rc:%x err:%x\n",
4243                                    rc, err);
4244                         return -1;
4245
4246                 default:
4247                         netdev_err(bp->dev, "Invalid ring\n");
4248                         return -1;
4249                 }
4250         }
4251         ring->fw_ring_id = ring_id;
4252         return rc;
4253 }
4254
4255 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
4256 {
4257         int rc;
4258
4259         if (BNXT_PF(bp)) {
4260                 struct hwrm_func_cfg_input req = {0};
4261
4262                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
4263                 req.fid = cpu_to_le16(0xffff);
4264                 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
4265                 req.async_event_cr = cpu_to_le16(idx);
4266                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4267         } else {
4268                 struct hwrm_func_vf_cfg_input req = {0};
4269
4270                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
4271                 req.enables =
4272                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
4273                 req.async_event_cr = cpu_to_le16(idx);
4274                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4275         }
4276         return rc;
4277 }
4278
4279 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
4280 {
4281         int i, rc = 0;
4282
4283         for (i = 0; i < bp->cp_nr_rings; i++) {
4284                 struct bnxt_napi *bnapi = bp->bnapi[i];
4285                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4286                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4287
4288                 cpr->cp_doorbell = bp->bar1 + i * 0x80;
4289                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL, i,
4290                                               INVALID_STATS_CTX_ID);
4291                 if (rc)
4292                         goto err_out;
4293                 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
4294                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
4295
4296                 if (!i) {
4297                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
4298                         if (rc)
4299                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
4300                 }
4301         }
4302
4303         for (i = 0; i < bp->tx_nr_rings; i++) {
4304                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4305                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
4306                 u32 map_idx = txr->bnapi->index;
4307                 u16 fw_stats_ctx = bp->grp_info[map_idx].fw_stats_ctx;
4308
4309                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX,
4310                                               map_idx, fw_stats_ctx);
4311                 if (rc)
4312                         goto err_out;
4313                 txr->tx_doorbell = bp->bar1 + map_idx * 0x80;
4314         }
4315
4316         for (i = 0; i < bp->rx_nr_rings; i++) {
4317                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4318                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
4319                 u32 map_idx = rxr->bnapi->index;
4320
4321                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX,
4322                                               map_idx, INVALID_STATS_CTX_ID);
4323                 if (rc)
4324                         goto err_out;
4325                 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80;
4326                 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
4327                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
4328         }
4329
4330         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
4331                 for (i = 0; i < bp->rx_nr_rings; i++) {
4332                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4333                         struct bnxt_ring_struct *ring =
4334                                                 &rxr->rx_agg_ring_struct;
4335                         u32 grp_idx = rxr->bnapi->index;
4336                         u32 map_idx = grp_idx + bp->rx_nr_rings;
4337
4338                         rc = hwrm_ring_alloc_send_msg(bp, ring,
4339                                                       HWRM_RING_ALLOC_AGG,
4340                                                       map_idx,
4341                                                       INVALID_STATS_CTX_ID);
4342                         if (rc)
4343                                 goto err_out;
4344
4345                         rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80;
4346                         writel(DB_KEY_RX | rxr->rx_agg_prod,
4347                                rxr->rx_agg_doorbell);
4348                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
4349                 }
4350         }
4351 err_out:
4352         return rc;
4353 }
4354
4355 static int hwrm_ring_free_send_msg(struct bnxt *bp,
4356                                    struct bnxt_ring_struct *ring,
4357                                    u32 ring_type, int cmpl_ring_id)
4358 {
4359         int rc;
4360         struct hwrm_ring_free_input req = {0};
4361         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
4362         u16 error_code;
4363
4364         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
4365         req.ring_type = ring_type;
4366         req.ring_id = cpu_to_le16(ring->fw_ring_id);
4367
4368         mutex_lock(&bp->hwrm_cmd_lock);
4369         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4370         error_code = le16_to_cpu(resp->error_code);
4371         mutex_unlock(&bp->hwrm_cmd_lock);
4372
4373         if (rc || error_code) {
4374                 switch (ring_type) {
4375                 case RING_FREE_REQ_RING_TYPE_L2_CMPL:
4376                         netdev_err(bp->dev, "hwrm_ring_free cp failed. rc:%d\n",
4377                                    rc);
4378                         return rc;
4379                 case RING_FREE_REQ_RING_TYPE_RX:
4380                         netdev_err(bp->dev, "hwrm_ring_free rx failed. rc:%d\n",
4381                                    rc);
4382                         return rc;
4383                 case RING_FREE_REQ_RING_TYPE_TX:
4384                         netdev_err(bp->dev, "hwrm_ring_free tx failed. rc:%d\n",
4385                                    rc);
4386                         return rc;
4387                 default:
4388                         netdev_err(bp->dev, "Invalid ring\n");
4389                         return -1;
4390                 }
4391         }
4392         return 0;
4393 }
4394
4395 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
4396 {
4397         int i;
4398
4399         if (!bp->bnapi)
4400                 return;
4401
4402         for (i = 0; i < bp->tx_nr_rings; i++) {
4403                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4404                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
4405                 u32 grp_idx = txr->bnapi->index;
4406                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
4407
4408                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4409                         hwrm_ring_free_send_msg(bp, ring,
4410                                                 RING_FREE_REQ_RING_TYPE_TX,
4411                                                 close_path ? cmpl_ring_id :
4412                                                 INVALID_HW_RING_ID);
4413                         ring->fw_ring_id = INVALID_HW_RING_ID;
4414                 }
4415         }
4416
4417         for (i = 0; i < bp->rx_nr_rings; i++) {
4418                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4419                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
4420                 u32 grp_idx = rxr->bnapi->index;
4421                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
4422
4423                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4424                         hwrm_ring_free_send_msg(bp, ring,
4425                                                 RING_FREE_REQ_RING_TYPE_RX,
4426                                                 close_path ? cmpl_ring_id :
4427                                                 INVALID_HW_RING_ID);
4428                         ring->fw_ring_id = INVALID_HW_RING_ID;
4429                         bp->grp_info[grp_idx].rx_fw_ring_id =
4430                                 INVALID_HW_RING_ID;
4431                 }
4432         }
4433
4434         for (i = 0; i < bp->rx_nr_rings; i++) {
4435                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4436                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
4437                 u32 grp_idx = rxr->bnapi->index;
4438                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
4439
4440                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4441                         hwrm_ring_free_send_msg(bp, ring,
4442                                                 RING_FREE_REQ_RING_TYPE_RX,
4443                                                 close_path ? cmpl_ring_id :
4444                                                 INVALID_HW_RING_ID);
4445                         ring->fw_ring_id = INVALID_HW_RING_ID;
4446                         bp->grp_info[grp_idx].agg_fw_ring_id =
4447                                 INVALID_HW_RING_ID;
4448                 }
4449         }
4450
4451         /* The completion rings are about to be freed.  After that the
4452          * IRQ doorbell will not work anymore.  So we need to disable
4453          * IRQ here.
4454          */
4455         bnxt_disable_int_sync(bp);
4456
4457         for (i = 0; i < bp->cp_nr_rings; i++) {
4458                 struct bnxt_napi *bnapi = bp->bnapi[i];
4459                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4460                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4461
4462                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4463                         hwrm_ring_free_send_msg(bp, ring,
4464                                                 RING_FREE_REQ_RING_TYPE_L2_CMPL,
4465                                                 INVALID_HW_RING_ID);
4466                         ring->fw_ring_id = INVALID_HW_RING_ID;
4467                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
4468                 }
4469         }
4470 }
4471
4472 /* Caller must hold bp->hwrm_cmd_lock */
4473 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
4474 {
4475         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
4476         struct hwrm_func_qcfg_input req = {0};
4477         int rc;
4478
4479         if (bp->hwrm_spec_code < 0x10601)
4480                 return 0;
4481
4482         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
4483         req.fid = cpu_to_le16(fid);
4484         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4485         if (!rc)
4486                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
4487
4488         return rc;
4489 }
4490
4491 static int bnxt_hwrm_reserve_tx_rings(struct bnxt *bp, int *tx_rings)
4492 {
4493         struct hwrm_func_cfg_input req = {0};
4494         int rc;
4495
4496         if (bp->hwrm_spec_code < 0x10601)
4497                 return 0;
4498
4499         if (BNXT_VF(bp))
4500                 return 0;
4501
4502         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
4503         req.fid = cpu_to_le16(0xffff);
4504         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS);
4505         req.num_tx_rings = cpu_to_le16(*tx_rings);
4506         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4507         if (rc)
4508                 return rc;
4509
4510         mutex_lock(&bp->hwrm_cmd_lock);
4511         rc = __bnxt_hwrm_get_tx_rings(bp, 0xffff, tx_rings);
4512         mutex_unlock(&bp->hwrm_cmd_lock);
4513         if (!rc)
4514                 bp->tx_reserved_rings = *tx_rings;
4515         return rc;
4516 }
4517
4518 static int bnxt_hwrm_check_tx_rings(struct bnxt *bp, int tx_rings)
4519 {
4520         struct hwrm_func_cfg_input req = {0};
4521         int rc;
4522
4523         if (bp->hwrm_spec_code < 0x10801)
4524                 return 0;
4525
4526         if (BNXT_VF(bp))
4527                 return 0;
4528
4529         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
4530         req.fid = cpu_to_le16(0xffff);
4531         req.flags = cpu_to_le32(FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST);
4532         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS);
4533         req.num_tx_rings = cpu_to_le16(tx_rings);
4534         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4535         if (rc)
4536                 return -ENOMEM;
4537         return 0;
4538 }
4539
4540 static void bnxt_hwrm_set_coal_params(struct bnxt_coal *hw_coal,
4541         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
4542 {
4543         u16 val, tmr, max, flags;
4544
4545         max = hw_coal->bufs_per_record * 128;
4546         if (hw_coal->budget)
4547                 max = hw_coal->bufs_per_record * hw_coal->budget;
4548
4549         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
4550         req->num_cmpl_aggr_int = cpu_to_le16(val);
4551
4552         /* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */
4553         val = min_t(u16, val, 63);
4554         req->num_cmpl_dma_aggr = cpu_to_le16(val);
4555
4556         /* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */
4557         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 63);
4558         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
4559
4560         tmr = BNXT_USEC_TO_COAL_TIMER(hw_coal->coal_ticks);
4561         tmr = max_t(u16, tmr, 1);
4562         req->int_lat_tmr_max = cpu_to_le16(tmr);
4563
4564         /* min timer set to 1/2 of interrupt timer */
4565         val = tmr / 2;
4566         req->int_lat_tmr_min = cpu_to_le16(val);
4567
4568         /* buf timer set to 1/4 of interrupt timer */
4569         val = max_t(u16, tmr / 4, 1);
4570         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
4571
4572         tmr = BNXT_USEC_TO_COAL_TIMER(hw_coal->coal_ticks_irq);
4573         tmr = max_t(u16, tmr, 1);
4574         req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(tmr);
4575
4576         flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
4577         if (hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
4578                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
4579         req->flags = cpu_to_le16(flags);
4580 }
4581
4582 int bnxt_hwrm_set_coal(struct bnxt *bp)
4583 {
4584         int i, rc = 0;
4585         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
4586                                                            req_tx = {0}, *req;
4587
4588         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
4589                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
4590         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
4591                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
4592
4593         bnxt_hwrm_set_coal_params(&bp->rx_coal, &req_rx);
4594         bnxt_hwrm_set_coal_params(&bp->tx_coal, &req_tx);
4595
4596         mutex_lock(&bp->hwrm_cmd_lock);
4597         for (i = 0; i < bp->cp_nr_rings; i++) {
4598                 struct bnxt_napi *bnapi = bp->bnapi[i];
4599
4600                 req = &req_rx;
4601                 if (!bnapi->rx_ring)
4602                         req = &req_tx;
4603                 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id);
4604
4605                 rc = _hwrm_send_message(bp, req, sizeof(*req),
4606                                         HWRM_CMD_TIMEOUT);
4607                 if (rc)
4608                         break;
4609         }
4610         mutex_unlock(&bp->hwrm_cmd_lock);
4611         return rc;
4612 }
4613
4614 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
4615 {
4616         int rc = 0, i;
4617         struct hwrm_stat_ctx_free_input req = {0};
4618
4619         if (!bp->bnapi)
4620                 return 0;
4621
4622         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4623                 return 0;
4624
4625         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
4626
4627         mutex_lock(&bp->hwrm_cmd_lock);
4628         for (i = 0; i < bp->cp_nr_rings; i++) {
4629                 struct bnxt_napi *bnapi = bp->bnapi[i];
4630                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4631
4632                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
4633                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
4634
4635                         rc = _hwrm_send_message(bp, &req, sizeof(req),
4636                                                 HWRM_CMD_TIMEOUT);
4637                         if (rc)
4638                                 break;
4639
4640                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4641                 }
4642         }
4643         mutex_unlock(&bp->hwrm_cmd_lock);
4644         return rc;
4645 }
4646
4647 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
4648 {
4649         int rc = 0, i;
4650         struct hwrm_stat_ctx_alloc_input req = {0};
4651         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4652
4653         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4654                 return 0;
4655
4656         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
4657
4658         req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
4659
4660         mutex_lock(&bp->hwrm_cmd_lock);
4661         for (i = 0; i < bp->cp_nr_rings; i++) {
4662                 struct bnxt_napi *bnapi = bp->bnapi[i];
4663                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4664
4665                 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
4666
4667                 rc = _hwrm_send_message(bp, &req, sizeof(req),
4668                                         HWRM_CMD_TIMEOUT);
4669                 if (rc)
4670                         break;
4671
4672                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
4673
4674                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
4675         }
4676         mutex_unlock(&bp->hwrm_cmd_lock);
4677         return rc;
4678 }
4679
4680 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
4681 {
4682         struct hwrm_func_qcfg_input req = {0};
4683         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
4684         u16 flags;
4685         int rc;
4686
4687         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
4688         req.fid = cpu_to_le16(0xffff);
4689         mutex_lock(&bp->hwrm_cmd_lock);
4690         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4691         if (rc)
4692                 goto func_qcfg_exit;
4693
4694 #ifdef CONFIG_BNXT_SRIOV
4695         if (BNXT_VF(bp)) {
4696                 struct bnxt_vf_info *vf = &bp->vf;
4697
4698                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
4699         }
4700 #endif
4701         flags = le16_to_cpu(resp->flags);
4702         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
4703                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
4704                 bp->flags |= BNXT_FLAG_FW_LLDP_AGENT;
4705                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
4706                         bp->flags |= BNXT_FLAG_FW_DCBX_AGENT;
4707         }
4708         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
4709                 bp->flags |= BNXT_FLAG_MULTI_HOST;
4710
4711         switch (resp->port_partition_type) {
4712         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
4713         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
4714         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
4715                 bp->port_partition_type = resp->port_partition_type;
4716                 break;
4717         }
4718         if (bp->hwrm_spec_code < 0x10707 ||
4719             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
4720                 bp->br_mode = BRIDGE_MODE_VEB;
4721         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
4722                 bp->br_mode = BRIDGE_MODE_VEPA;
4723         else
4724                 bp->br_mode = BRIDGE_MODE_UNDEF;
4725
4726         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
4727         if (!bp->max_mtu)
4728                 bp->max_mtu = BNXT_MAX_MTU;
4729
4730 func_qcfg_exit:
4731         mutex_unlock(&bp->hwrm_cmd_lock);
4732         return rc;
4733 }
4734
4735 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
4736 {
4737         int rc = 0;
4738         struct hwrm_func_qcaps_input req = {0};
4739         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4740
4741         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
4742         req.fid = cpu_to_le16(0xffff);
4743
4744         mutex_lock(&bp->hwrm_cmd_lock);
4745         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4746         if (rc)
4747                 goto hwrm_func_qcaps_exit;
4748
4749         if (resp->flags & cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED))
4750                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
4751         if (resp->flags & cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED))
4752                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
4753
4754         bp->tx_push_thresh = 0;
4755         if (resp->flags &
4756             cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED))
4757                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
4758
4759         if (BNXT_PF(bp)) {
4760                 struct bnxt_pf_info *pf = &bp->pf;
4761
4762                 pf->fw_fid = le16_to_cpu(resp->fid);
4763                 pf->port_id = le16_to_cpu(resp->port_id);
4764                 bp->dev->dev_port = pf->port_id;
4765                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
4766                 pf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
4767                 pf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
4768                 pf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
4769                 pf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
4770                 pf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
4771                 if (!pf->max_hw_ring_grps)
4772                         pf->max_hw_ring_grps = pf->max_tx_rings;
4773                 pf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
4774                 pf->max_vnics = le16_to_cpu(resp->max_vnics);
4775                 pf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
4776                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
4777                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
4778                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
4779                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
4780                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
4781                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
4782                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
4783                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
4784                 if (resp->flags &
4785                     cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED))
4786                         bp->flags |= BNXT_FLAG_WOL_CAP;
4787         } else {
4788 #ifdef CONFIG_BNXT_SRIOV
4789                 struct bnxt_vf_info *vf = &bp->vf;
4790
4791                 vf->fw_fid = le16_to_cpu(resp->fid);
4792
4793                 vf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
4794                 vf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
4795                 vf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
4796                 vf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
4797                 vf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
4798                 if (!vf->max_hw_ring_grps)
4799                         vf->max_hw_ring_grps = vf->max_tx_rings;
4800                 vf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
4801                 vf->max_vnics = le16_to_cpu(resp->max_vnics);
4802                 vf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
4803
4804                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
4805 #endif
4806         }
4807
4808 hwrm_func_qcaps_exit:
4809         mutex_unlock(&bp->hwrm_cmd_lock);
4810         return rc;
4811 }
4812
4813 static int bnxt_hwrm_func_reset(struct bnxt *bp)
4814 {
4815         struct hwrm_func_reset_input req = {0};
4816
4817         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
4818         req.enables = 0;
4819
4820         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
4821 }
4822
4823 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
4824 {
4825         int rc = 0;
4826         struct hwrm_queue_qportcfg_input req = {0};
4827         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
4828         u8 i, *qptr;
4829
4830         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
4831
4832         mutex_lock(&bp->hwrm_cmd_lock);
4833         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4834         if (rc)
4835                 goto qportcfg_exit;
4836
4837         if (!resp->max_configurable_queues) {
4838                 rc = -EINVAL;
4839                 goto qportcfg_exit;
4840         }
4841         bp->max_tc = resp->max_configurable_queues;
4842         bp->max_lltc = resp->max_configurable_lossless_queues;
4843         if (bp->max_tc > BNXT_MAX_QUEUE)
4844                 bp->max_tc = BNXT_MAX_QUEUE;
4845
4846         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
4847                 bp->max_tc = 1;
4848
4849         if (bp->max_lltc > bp->max_tc)
4850                 bp->max_lltc = bp->max_tc;
4851
4852         qptr = &resp->queue_id0;
4853         for (i = 0; i < bp->max_tc; i++) {
4854                 bp->q_info[i].queue_id = *qptr++;
4855                 bp->q_info[i].queue_profile = *qptr++;
4856         }
4857
4858 qportcfg_exit:
4859         mutex_unlock(&bp->hwrm_cmd_lock);
4860         return rc;
4861 }
4862
4863 static int bnxt_hwrm_ver_get(struct bnxt *bp)
4864 {
4865         int rc;
4866         struct hwrm_ver_get_input req = {0};
4867         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
4868         u32 dev_caps_cfg;
4869
4870         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
4871         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
4872         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
4873         req.hwrm_intf_min = HWRM_VERSION_MINOR;
4874         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
4875         mutex_lock(&bp->hwrm_cmd_lock);
4876         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4877         if (rc)
4878                 goto hwrm_ver_get_exit;
4879
4880         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
4881
4882         bp->hwrm_spec_code = resp->hwrm_intf_maj << 16 |
4883                              resp->hwrm_intf_min << 8 | resp->hwrm_intf_upd;
4884         if (resp->hwrm_intf_maj < 1) {
4885                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
4886                             resp->hwrm_intf_maj, resp->hwrm_intf_min,
4887                             resp->hwrm_intf_upd);
4888                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
4889         }
4890         snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d",
4891                  resp->hwrm_fw_maj, resp->hwrm_fw_min, resp->hwrm_fw_bld,
4892                  resp->hwrm_fw_rsvd);
4893
4894         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
4895         if (!bp->hwrm_cmd_timeout)
4896                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
4897
4898         if (resp->hwrm_intf_maj >= 1)
4899                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
4900
4901         bp->chip_num = le16_to_cpu(resp->chip_num);
4902         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
4903             !resp->chip_metal)
4904                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
4905
4906         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
4907         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
4908             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
4909                 bp->flags |= BNXT_FLAG_SHORT_CMD;
4910
4911 hwrm_ver_get_exit:
4912         mutex_unlock(&bp->hwrm_cmd_lock);
4913         return rc;
4914 }
4915
4916 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
4917 {
4918         struct hwrm_fw_set_time_input req = {0};
4919         struct tm tm;
4920         time64_t now = ktime_get_real_seconds();
4921
4922         if (bp->hwrm_spec_code < 0x10400)
4923                 return -EOPNOTSUPP;
4924
4925         time64_to_tm(now, 0, &tm);
4926         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
4927         req.year = cpu_to_le16(1900 + tm.tm_year);
4928         req.month = 1 + tm.tm_mon;
4929         req.day = tm.tm_mday;
4930         req.hour = tm.tm_hour;
4931         req.minute = tm.tm_min;
4932         req.second = tm.tm_sec;
4933         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4934 }
4935
4936 static int bnxt_hwrm_port_qstats(struct bnxt *bp)
4937 {
4938         int rc;
4939         struct bnxt_pf_info *pf = &bp->pf;
4940         struct hwrm_port_qstats_input req = {0};
4941
4942         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
4943                 return 0;
4944
4945         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
4946         req.port_id = cpu_to_le16(pf->port_id);
4947         req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
4948         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
4949         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4950         return rc;
4951 }
4952
4953 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
4954 {
4955         if (bp->vxlan_port_cnt) {
4956                 bnxt_hwrm_tunnel_dst_port_free(
4957                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
4958         }
4959         bp->vxlan_port_cnt = 0;
4960         if (bp->nge_port_cnt) {
4961                 bnxt_hwrm_tunnel_dst_port_free(
4962                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
4963         }
4964         bp->nge_port_cnt = 0;
4965 }
4966
4967 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
4968 {
4969         int rc, i;
4970         u32 tpa_flags = 0;
4971
4972         if (set_tpa)
4973                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
4974         for (i = 0; i < bp->nr_vnics; i++) {
4975                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
4976                 if (rc) {
4977                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
4978                                    i, rc);
4979                         return rc;
4980                 }
4981         }
4982         return 0;
4983 }
4984
4985 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
4986 {
4987         int i;
4988
4989         for (i = 0; i < bp->nr_vnics; i++)
4990                 bnxt_hwrm_vnic_set_rss(bp, i, false);
4991 }
4992
4993 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
4994                                     bool irq_re_init)
4995 {
4996         if (bp->vnic_info) {
4997                 bnxt_hwrm_clear_vnic_filter(bp);
4998                 /* clear all RSS setting before free vnic ctx */
4999                 bnxt_hwrm_clear_vnic_rss(bp);
5000                 bnxt_hwrm_vnic_ctx_free(bp);
5001                 /* before free the vnic, undo the vnic tpa settings */
5002                 if (bp->flags & BNXT_FLAG_TPA)
5003                         bnxt_set_tpa(bp, false);
5004                 bnxt_hwrm_vnic_free(bp);
5005         }
5006         bnxt_hwrm_ring_free(bp, close_path);
5007         bnxt_hwrm_ring_grp_free(bp);
5008         if (irq_re_init) {
5009                 bnxt_hwrm_stat_ctx_free(bp);
5010                 bnxt_hwrm_free_tunnel_ports(bp);
5011         }
5012 }
5013
5014 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
5015 {
5016         struct hwrm_func_cfg_input req = {0};
5017         int rc;
5018
5019         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5020         req.fid = cpu_to_le16(0xffff);
5021         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
5022         if (br_mode == BRIDGE_MODE_VEB)
5023                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
5024         else if (br_mode == BRIDGE_MODE_VEPA)
5025                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
5026         else
5027                 return -EINVAL;
5028         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5029         if (rc)
5030                 rc = -EIO;
5031         return rc;
5032 }
5033
5034 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
5035 {
5036         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5037         int rc;
5038
5039         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
5040                 goto skip_rss_ctx;
5041
5042         /* allocate context for vnic */
5043         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
5044         if (rc) {
5045                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
5046                            vnic_id, rc);
5047                 goto vnic_setup_err;
5048         }
5049         bp->rsscos_nr_ctxs++;
5050
5051         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
5052                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
5053                 if (rc) {
5054                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
5055                                    vnic_id, rc);
5056                         goto vnic_setup_err;
5057                 }
5058                 bp->rsscos_nr_ctxs++;
5059         }
5060
5061 skip_rss_ctx:
5062         /* configure default vnic, ring grp */
5063         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
5064         if (rc) {
5065                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
5066                            vnic_id, rc);
5067                 goto vnic_setup_err;
5068         }
5069
5070         /* Enable RSS hashing on vnic */
5071         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
5072         if (rc) {
5073                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
5074                            vnic_id, rc);
5075                 goto vnic_setup_err;
5076         }
5077
5078         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
5079                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
5080                 if (rc) {
5081                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
5082                                    vnic_id, rc);
5083                 }
5084         }
5085
5086 vnic_setup_err:
5087         return rc;
5088 }
5089
5090 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
5091 {
5092 #ifdef CONFIG_RFS_ACCEL
5093         int i, rc = 0;
5094
5095         for (i = 0; i < bp->rx_nr_rings; i++) {
5096                 struct bnxt_vnic_info *vnic;
5097                 u16 vnic_id = i + 1;
5098                 u16 ring_id = i;
5099
5100                 if (vnic_id >= bp->nr_vnics)
5101                         break;
5102
5103                 vnic = &bp->vnic_info[vnic_id];
5104                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
5105                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
5106                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
5107                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
5108                 if (rc) {
5109                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
5110                                    vnic_id, rc);
5111                         break;
5112                 }
5113                 rc = bnxt_setup_vnic(bp, vnic_id);
5114                 if (rc)
5115                         break;
5116         }
5117         return rc;
5118 #else
5119         return 0;
5120 #endif
5121 }
5122
5123 /* Allow PF and VF with default VLAN to be in promiscuous mode */
5124 static bool bnxt_promisc_ok(struct bnxt *bp)
5125 {
5126 #ifdef CONFIG_BNXT_SRIOV
5127         if (BNXT_VF(bp) && !bp->vf.vlan)
5128                 return false;
5129 #endif
5130         return true;
5131 }
5132
5133 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
5134 {
5135         unsigned int rc = 0;
5136
5137         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
5138         if (rc) {
5139                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
5140                            rc);
5141                 return rc;
5142         }
5143
5144         rc = bnxt_hwrm_vnic_cfg(bp, 1);
5145         if (rc) {
5146                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
5147                            rc);
5148                 return rc;
5149         }
5150         return rc;
5151 }
5152
5153 static int bnxt_cfg_rx_mode(struct bnxt *);
5154 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
5155
5156 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
5157 {
5158         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5159         int rc = 0;
5160         unsigned int rx_nr_rings = bp->rx_nr_rings;
5161
5162         if (irq_re_init) {
5163                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
5164                 if (rc) {
5165                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
5166                                    rc);
5167                         goto err_out;
5168                 }
5169                 if (bp->tx_reserved_rings != bp->tx_nr_rings) {
5170                         int tx = bp->tx_nr_rings;
5171
5172                         if (bnxt_hwrm_reserve_tx_rings(bp, &tx) ||
5173                             tx < bp->tx_nr_rings) {
5174                                 rc = -ENOMEM;
5175                                 goto err_out;
5176                         }
5177                 }
5178         }
5179
5180         rc = bnxt_hwrm_ring_alloc(bp);
5181         if (rc) {
5182                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
5183                 goto err_out;
5184         }
5185
5186         rc = bnxt_hwrm_ring_grp_alloc(bp);
5187         if (rc) {
5188                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
5189                 goto err_out;
5190         }
5191
5192         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5193                 rx_nr_rings--;
5194
5195         /* default vnic 0 */
5196         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
5197         if (rc) {
5198                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
5199                 goto err_out;
5200         }
5201
5202         rc = bnxt_setup_vnic(bp, 0);
5203         if (rc)
5204                 goto err_out;
5205
5206         if (bp->flags & BNXT_FLAG_RFS) {
5207                 rc = bnxt_alloc_rfs_vnics(bp);
5208                 if (rc)
5209                         goto err_out;
5210         }
5211
5212         if (bp->flags & BNXT_FLAG_TPA) {
5213                 rc = bnxt_set_tpa(bp, true);
5214                 if (rc)
5215                         goto err_out;
5216         }
5217
5218         if (BNXT_VF(bp))
5219                 bnxt_update_vf_mac(bp);
5220
5221         /* Filter for default vnic 0 */
5222         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
5223         if (rc) {
5224                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
5225                 goto err_out;
5226         }
5227         vnic->uc_filter_count = 1;
5228
5229         vnic->rx_mask = CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
5230
5231         if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
5232                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5233
5234         if (bp->dev->flags & IFF_ALLMULTI) {
5235                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5236                 vnic->mc_list_count = 0;
5237         } else {
5238                 u32 mask = 0;
5239
5240                 bnxt_mc_list_updated(bp, &mask);
5241                 vnic->rx_mask |= mask;
5242         }
5243
5244         rc = bnxt_cfg_rx_mode(bp);
5245         if (rc)
5246                 goto err_out;
5247
5248         rc = bnxt_hwrm_set_coal(bp);
5249         if (rc)
5250                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
5251                                 rc);
5252
5253         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
5254                 rc = bnxt_setup_nitroa0_vnic(bp);
5255                 if (rc)
5256                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
5257                                    rc);
5258         }
5259
5260         if (BNXT_VF(bp)) {
5261                 bnxt_hwrm_func_qcfg(bp);
5262                 netdev_update_features(bp->dev);
5263         }
5264
5265         return 0;
5266
5267 err_out:
5268         bnxt_hwrm_resource_free(bp, 0, true);
5269
5270         return rc;
5271 }
5272
5273 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
5274 {
5275         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
5276         return 0;
5277 }
5278
5279 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
5280 {
5281         bnxt_init_cp_rings(bp);
5282         bnxt_init_rx_rings(bp);
5283         bnxt_init_tx_rings(bp);
5284         bnxt_init_ring_grps(bp, irq_re_init);
5285         bnxt_init_vnics(bp);
5286
5287         return bnxt_init_chip(bp, irq_re_init);
5288 }
5289
5290 static int bnxt_set_real_num_queues(struct bnxt *bp)
5291 {
5292         int rc;
5293         struct net_device *dev = bp->dev;
5294
5295         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
5296                                           bp->tx_nr_rings_xdp);
5297         if (rc)
5298                 return rc;
5299
5300         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
5301         if (rc)
5302                 return rc;
5303
5304 #ifdef CONFIG_RFS_ACCEL
5305         if (bp->flags & BNXT_FLAG_RFS)
5306                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
5307 #endif
5308
5309         return rc;
5310 }
5311
5312 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5313                            bool shared)
5314 {
5315         int _rx = *rx, _tx = *tx;
5316
5317         if (shared) {
5318                 *rx = min_t(int, _rx, max);
5319                 *tx = min_t(int, _tx, max);
5320         } else {
5321                 if (max < 2)
5322                         return -ENOMEM;
5323
5324                 while (_rx + _tx > max) {
5325                         if (_rx > _tx && _rx > 1)
5326                                 _rx--;
5327                         else if (_tx > 1)
5328                                 _tx--;
5329                 }
5330                 *rx = _rx;
5331                 *tx = _tx;
5332         }
5333         return 0;
5334 }
5335
5336 static void bnxt_setup_msix(struct bnxt *bp)
5337 {
5338         const int len = sizeof(bp->irq_tbl[0].name);
5339         struct net_device *dev = bp->dev;
5340         int tcs, i;
5341
5342         tcs = netdev_get_num_tc(dev);
5343         if (tcs > 1) {
5344                 int i, off, count;
5345
5346                 for (i = 0; i < tcs; i++) {
5347                         count = bp->tx_nr_rings_per_tc;
5348                         off = i * count;
5349                         netdev_set_tc_queue(dev, i, count, off);
5350                 }
5351         }
5352
5353         for (i = 0; i < bp->cp_nr_rings; i++) {
5354                 char *attr;
5355
5356                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5357                         attr = "TxRx";
5358                 else if (i < bp->rx_nr_rings)
5359                         attr = "rx";
5360                 else
5361                         attr = "tx";
5362
5363                 snprintf(bp->irq_tbl[i].name, len, "%s-%s-%d", dev->name, attr,
5364                          i);
5365                 bp->irq_tbl[i].handler = bnxt_msix;
5366         }
5367 }
5368
5369 static void bnxt_setup_inta(struct bnxt *bp)
5370 {
5371         const int len = sizeof(bp->irq_tbl[0].name);
5372
5373         if (netdev_get_num_tc(bp->dev))
5374                 netdev_reset_tc(bp->dev);
5375
5376         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
5377                  0);
5378         bp->irq_tbl[0].handler = bnxt_inta;
5379 }
5380
5381 static int bnxt_setup_int_mode(struct bnxt *bp)
5382 {
5383         int rc;
5384
5385         if (bp->flags & BNXT_FLAG_USING_MSIX)
5386                 bnxt_setup_msix(bp);
5387         else
5388                 bnxt_setup_inta(bp);
5389
5390         rc = bnxt_set_real_num_queues(bp);
5391         return rc;
5392 }
5393
5394 #ifdef CONFIG_RFS_ACCEL
5395 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
5396 {
5397 #if defined(CONFIG_BNXT_SRIOV)
5398         if (BNXT_VF(bp))
5399                 return bp->vf.max_rsscos_ctxs;
5400 #endif
5401         return bp->pf.max_rsscos_ctxs;
5402 }
5403
5404 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
5405 {
5406 #if defined(CONFIG_BNXT_SRIOV)
5407         if (BNXT_VF(bp))
5408                 return bp->vf.max_vnics;
5409 #endif
5410         return bp->pf.max_vnics;
5411 }
5412 #endif
5413
5414 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
5415 {
5416 #if defined(CONFIG_BNXT_SRIOV)
5417         if (BNXT_VF(bp))
5418                 return bp->vf.max_stat_ctxs;
5419 #endif
5420         return bp->pf.max_stat_ctxs;
5421 }
5422
5423 void bnxt_set_max_func_stat_ctxs(struct bnxt *bp, unsigned int max)
5424 {
5425 #if defined(CONFIG_BNXT_SRIOV)
5426         if (BNXT_VF(bp))
5427                 bp->vf.max_stat_ctxs = max;
5428         else
5429 #endif
5430                 bp->pf.max_stat_ctxs = max;
5431 }
5432
5433 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
5434 {
5435 #if defined(CONFIG_BNXT_SRIOV)
5436         if (BNXT_VF(bp))
5437                 return bp->vf.max_cp_rings;
5438 #endif
5439         return bp->pf.max_cp_rings;
5440 }
5441
5442 void bnxt_set_max_func_cp_rings(struct bnxt *bp, unsigned int max)
5443 {
5444 #if defined(CONFIG_BNXT_SRIOV)
5445         if (BNXT_VF(bp))
5446                 bp->vf.max_cp_rings = max;
5447         else
5448 #endif
5449                 bp->pf.max_cp_rings = max;
5450 }
5451
5452 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
5453 {
5454 #if defined(CONFIG_BNXT_SRIOV)
5455         if (BNXT_VF(bp))
5456                 return min_t(unsigned int, bp->vf.max_irqs,
5457                              bp->vf.max_cp_rings);
5458 #endif
5459         return min_t(unsigned int, bp->pf.max_irqs, bp->pf.max_cp_rings);
5460 }
5461
5462 void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
5463 {
5464 #if defined(CONFIG_BNXT_SRIOV)
5465         if (BNXT_VF(bp))
5466                 bp->vf.max_irqs = max_irqs;
5467         else
5468 #endif
5469                 bp->pf.max_irqs = max_irqs;
5470 }
5471
5472 static int bnxt_init_msix(struct bnxt *bp)
5473 {
5474         int i, total_vecs, rc = 0, min = 1;
5475         struct msix_entry *msix_ent;
5476
5477         total_vecs = bnxt_get_max_func_irqs(bp);
5478         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
5479         if (!msix_ent)
5480                 return -ENOMEM;
5481
5482         for (i = 0; i < total_vecs; i++) {
5483                 msix_ent[i].entry = i;
5484                 msix_ent[i].vector = 0;
5485         }
5486
5487         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
5488                 min = 2;
5489
5490         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
5491         if (total_vecs < 0) {
5492                 rc = -ENODEV;
5493                 goto msix_setup_exit;
5494         }
5495
5496         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
5497         if (bp->irq_tbl) {
5498                 for (i = 0; i < total_vecs; i++)
5499                         bp->irq_tbl[i].vector = msix_ent[i].vector;
5500
5501                 bp->total_irqs = total_vecs;
5502                 /* Trim rings based upon num of vectors allocated */
5503                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
5504                                      total_vecs, min == 1);
5505                 if (rc)
5506                         goto msix_setup_exit;
5507
5508                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
5509                 bp->cp_nr_rings = (min == 1) ?
5510                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
5511                                   bp->tx_nr_rings + bp->rx_nr_rings;
5512
5513         } else {
5514                 rc = -ENOMEM;
5515                 goto msix_setup_exit;
5516         }
5517         bp->flags |= BNXT_FLAG_USING_MSIX;
5518         kfree(msix_ent);
5519         return 0;
5520
5521 msix_setup_exit:
5522         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
5523         kfree(bp->irq_tbl);
5524         bp->irq_tbl = NULL;
5525         pci_disable_msix(bp->pdev);
5526         kfree(msix_ent);
5527         return rc;
5528 }
5529
5530 static int bnxt_init_inta(struct bnxt *bp)
5531 {
5532         bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
5533         if (!bp->irq_tbl)
5534                 return -ENOMEM;
5535
5536         bp->total_irqs = 1;
5537         bp->rx_nr_rings = 1;
5538         bp->tx_nr_rings = 1;
5539         bp->cp_nr_rings = 1;
5540         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
5541         bp->flags |= BNXT_FLAG_SHARED_RINGS;
5542         bp->irq_tbl[0].vector = bp->pdev->irq;
5543         return 0;
5544 }
5545
5546 static int bnxt_init_int_mode(struct bnxt *bp)
5547 {
5548         int rc = 0;
5549
5550         if (bp->flags & BNXT_FLAG_MSIX_CAP)
5551                 rc = bnxt_init_msix(bp);
5552
5553         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
5554                 /* fallback to INTA */
5555                 rc = bnxt_init_inta(bp);
5556         }
5557         return rc;
5558 }
5559
5560 static void bnxt_clear_int_mode(struct bnxt *bp)
5561 {
5562         if (bp->flags & BNXT_FLAG_USING_MSIX)
5563                 pci_disable_msix(bp->pdev);
5564
5565         kfree(bp->irq_tbl);
5566         bp->irq_tbl = NULL;
5567         bp->flags &= ~BNXT_FLAG_USING_MSIX;
5568 }
5569
5570 static void bnxt_free_irq(struct bnxt *bp)
5571 {
5572         struct bnxt_irq *irq;
5573         int i;
5574
5575 #ifdef CONFIG_RFS_ACCEL
5576         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
5577         bp->dev->rx_cpu_rmap = NULL;
5578 #endif
5579         if (!bp->irq_tbl)
5580                 return;
5581
5582         for (i = 0; i < bp->cp_nr_rings; i++) {
5583                 irq = &bp->irq_tbl[i];
5584                 if (irq->requested) {
5585                         if (irq->have_cpumask) {
5586                                 irq_set_affinity_hint(irq->vector, NULL);
5587                                 free_cpumask_var(irq->cpu_mask);
5588                                 irq->have_cpumask = 0;
5589                         }
5590                         free_irq(irq->vector, bp->bnapi[i]);
5591                 }
5592
5593                 irq->requested = 0;
5594         }
5595 }
5596
5597 static int bnxt_request_irq(struct bnxt *bp)
5598 {
5599         int i, j, rc = 0;
5600         unsigned long flags = 0;
5601 #ifdef CONFIG_RFS_ACCEL
5602         struct cpu_rmap *rmap = bp->dev->rx_cpu_rmap;
5603 #endif
5604
5605         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
5606                 flags = IRQF_SHARED;
5607
5608         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
5609                 struct bnxt_irq *irq = &bp->irq_tbl[i];
5610 #ifdef CONFIG_RFS_ACCEL
5611                 if (rmap && bp->bnapi[i]->rx_ring) {
5612                         rc = irq_cpu_rmap_add(rmap, irq->vector);
5613                         if (rc)
5614                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
5615                                             j);
5616                         j++;
5617                 }
5618 #endif
5619                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
5620                                  bp->bnapi[i]);
5621                 if (rc)
5622                         break;
5623
5624                 irq->requested = 1;
5625
5626                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
5627                         int numa_node = dev_to_node(&bp->pdev->dev);
5628
5629                         irq->have_cpumask = 1;
5630                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
5631                                         irq->cpu_mask);
5632                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
5633                         if (rc) {
5634                                 netdev_warn(bp->dev,
5635                                             "Set affinity failed, IRQ = %d\n",
5636                                             irq->vector);
5637                                 break;
5638                         }
5639                 }
5640         }
5641         return rc;
5642 }
5643
5644 static void bnxt_del_napi(struct bnxt *bp)
5645 {
5646         int i;
5647
5648         if (!bp->bnapi)
5649                 return;
5650
5651         for (i = 0; i < bp->cp_nr_rings; i++) {
5652                 struct bnxt_napi *bnapi = bp->bnapi[i];
5653
5654                 napi_hash_del(&bnapi->napi);
5655                 netif_napi_del(&bnapi->napi);
5656         }
5657         /* We called napi_hash_del() before netif_napi_del(), we need
5658          * to respect an RCU grace period before freeing napi structures.
5659          */
5660         synchronize_net();
5661 }
5662
5663 static void bnxt_init_napi(struct bnxt *bp)
5664 {
5665         int i;
5666         unsigned int cp_nr_rings = bp->cp_nr_rings;
5667         struct bnxt_napi *bnapi;
5668
5669         if (bp->flags & BNXT_FLAG_USING_MSIX) {
5670                 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5671                         cp_nr_rings--;
5672                 for (i = 0; i < cp_nr_rings; i++) {
5673                         bnapi = bp->bnapi[i];
5674                         netif_napi_add(bp->dev, &bnapi->napi,
5675                                        bnxt_poll, 64);
5676                 }
5677                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
5678                         bnapi = bp->bnapi[cp_nr_rings];
5679                         netif_napi_add(bp->dev, &bnapi->napi,
5680                                        bnxt_poll_nitroa0, 64);
5681                 }
5682         } else {
5683                 bnapi = bp->bnapi[0];
5684                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
5685         }
5686 }
5687
5688 static void bnxt_disable_napi(struct bnxt *bp)
5689 {
5690         int i;
5691
5692         if (!bp->bnapi)
5693                 return;
5694
5695         for (i = 0; i < bp->cp_nr_rings; i++)
5696                 napi_disable(&bp->bnapi[i]->napi);
5697 }
5698
5699 static void bnxt_enable_napi(struct bnxt *bp)
5700 {
5701         int i;
5702
5703         for (i = 0; i < bp->cp_nr_rings; i++) {
5704                 bp->bnapi[i]->in_reset = false;
5705                 napi_enable(&bp->bnapi[i]->napi);
5706         }
5707 }
5708
5709 void bnxt_tx_disable(struct bnxt *bp)
5710 {
5711         int i;
5712         struct bnxt_tx_ring_info *txr;
5713
5714         if (bp->tx_ring) {
5715                 for (i = 0; i < bp->tx_nr_rings; i++) {
5716                         txr = &bp->tx_ring[i];
5717                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
5718                 }
5719         }
5720         /* Stop all TX queues */
5721         netif_tx_disable(bp->dev);
5722         netif_carrier_off(bp->dev);
5723 }
5724
5725 void bnxt_tx_enable(struct bnxt *bp)
5726 {
5727         int i;
5728         struct bnxt_tx_ring_info *txr;
5729
5730         for (i = 0; i < bp->tx_nr_rings; i++) {
5731                 txr = &bp->tx_ring[i];
5732                 txr->dev_state = 0;
5733         }
5734         netif_tx_wake_all_queues(bp->dev);
5735         if (bp->link_info.link_up)
5736                 netif_carrier_on(bp->dev);
5737 }
5738
5739 static void bnxt_report_link(struct bnxt *bp)
5740 {
5741         if (bp->link_info.link_up) {
5742                 const char *duplex;
5743                 const char *flow_ctrl;
5744                 u32 speed;
5745                 u16 fec;
5746
5747                 netif_carrier_on(bp->dev);
5748                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
5749                         duplex = "full";
5750                 else
5751                         duplex = "half";
5752                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
5753                         flow_ctrl = "ON - receive & transmit";
5754                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
5755                         flow_ctrl = "ON - transmit";
5756                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
5757                         flow_ctrl = "ON - receive";
5758                 else
5759                         flow_ctrl = "none";
5760                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
5761                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
5762                             speed, duplex, flow_ctrl);
5763                 if (bp->flags & BNXT_FLAG_EEE_CAP)
5764                         netdev_info(bp->dev, "EEE is %s\n",
5765                                     bp->eee.eee_active ? "active" :
5766                                                          "not active");
5767                 fec = bp->link_info.fec_cfg;
5768                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
5769                         netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
5770                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
5771                                     (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
5772                                      (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
5773         } else {
5774                 netif_carrier_off(bp->dev);
5775                 netdev_err(bp->dev, "NIC Link is Down\n");
5776         }
5777 }
5778
5779 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
5780 {
5781         int rc = 0;
5782         struct hwrm_port_phy_qcaps_input req = {0};
5783         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5784         struct bnxt_link_info *link_info = &bp->link_info;
5785
5786         if (bp->hwrm_spec_code < 0x10201)
5787                 return 0;
5788
5789         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
5790
5791         mutex_lock(&bp->hwrm_cmd_lock);
5792         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5793         if (rc)
5794                 goto hwrm_phy_qcaps_exit;
5795
5796         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
5797                 struct ethtool_eee *eee = &bp->eee;
5798                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
5799
5800                 bp->flags |= BNXT_FLAG_EEE_CAP;
5801                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
5802                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
5803                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
5804                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
5805                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
5806         }
5807         if (resp->supported_speeds_auto_mode)
5808                 link_info->support_auto_speeds =
5809                         le16_to_cpu(resp->supported_speeds_auto_mode);
5810
5811         bp->port_count = resp->port_cnt;
5812
5813 hwrm_phy_qcaps_exit:
5814         mutex_unlock(&bp->hwrm_cmd_lock);
5815         return rc;
5816 }
5817
5818 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
5819 {
5820         int rc = 0;
5821         struct bnxt_link_info *link_info = &bp->link_info;
5822         struct hwrm_port_phy_qcfg_input req = {0};
5823         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5824         u8 link_up = link_info->link_up;
5825         u16 diff;
5826
5827         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
5828
5829         mutex_lock(&bp->hwrm_cmd_lock);
5830         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5831         if (rc) {
5832                 mutex_unlock(&bp->hwrm_cmd_lock);
5833                 return rc;
5834         }
5835
5836         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
5837         link_info->phy_link_status = resp->link;
5838         link_info->duplex = resp->duplex_cfg;
5839         if (bp->hwrm_spec_code >= 0x10800)
5840                 link_info->duplex = resp->duplex_state;
5841         link_info->pause = resp->pause;
5842         link_info->auto_mode = resp->auto_mode;
5843         link_info->auto_pause_setting = resp->auto_pause;
5844         link_info->lp_pause = resp->link_partner_adv_pause;
5845         link_info->force_pause_setting = resp->force_pause;
5846         link_info->duplex_setting = resp->duplex_cfg;
5847         if (link_info->phy_link_status == BNXT_LINK_LINK)
5848                 link_info->link_speed = le16_to_cpu(resp->link_speed);
5849         else
5850                 link_info->link_speed = 0;
5851         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
5852         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
5853         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
5854         link_info->lp_auto_link_speeds =
5855                 le16_to_cpu(resp->link_partner_adv_speeds);
5856         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
5857         link_info->phy_ver[0] = resp->phy_maj;
5858         link_info->phy_ver[1] = resp->phy_min;
5859         link_info->phy_ver[2] = resp->phy_bld;
5860         link_info->media_type = resp->media_type;
5861         link_info->phy_type = resp->phy_type;
5862         link_info->transceiver = resp->xcvr_pkg_type;
5863         link_info->phy_addr = resp->eee_config_phy_addr &
5864                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
5865         link_info->module_status = resp->module_status;
5866
5867         if (bp->flags & BNXT_FLAG_EEE_CAP) {
5868                 struct ethtool_eee *eee = &bp->eee;
5869                 u16 fw_speeds;
5870
5871                 eee->eee_active = 0;
5872                 if (resp->eee_config_phy_addr &
5873                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
5874                         eee->eee_active = 1;
5875                         fw_speeds = le16_to_cpu(
5876                                 resp->link_partner_adv_eee_link_speed_mask);
5877                         eee->lp_advertised =
5878                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
5879                 }
5880
5881                 /* Pull initial EEE config */
5882                 if (!chng_link_state) {
5883                         if (resp->eee_config_phy_addr &
5884                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
5885                                 eee->eee_enabled = 1;
5886
5887                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
5888                         eee->advertised =
5889                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
5890
5891                         if (resp->eee_config_phy_addr &
5892                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
5893                                 __le32 tmr;
5894
5895                                 eee->tx_lpi_enabled = 1;
5896                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
5897                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
5898                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
5899                         }
5900                 }
5901         }
5902
5903         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
5904         if (bp->hwrm_spec_code >= 0x10504)
5905                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
5906
5907         /* TODO: need to add more logic to report VF link */
5908         if (chng_link_state) {
5909                 if (link_info->phy_link_status == BNXT_LINK_LINK)
5910                         link_info->link_up = 1;
5911                 else
5912                         link_info->link_up = 0;
5913                 if (link_up != link_info->link_up)
5914                         bnxt_report_link(bp);
5915         } else {
5916                 /* alwasy link down if not require to update link state */
5917                 link_info->link_up = 0;
5918         }
5919         mutex_unlock(&bp->hwrm_cmd_lock);
5920
5921         diff = link_info->support_auto_speeds ^ link_info->advertising;
5922         if ((link_info->support_auto_speeds | diff) !=
5923             link_info->support_auto_speeds) {
5924                 /* An advertised speed is no longer supported, so we need to
5925                  * update the advertisement settings.  Caller holds RTNL
5926                  * so we can modify link settings.
5927                  */
5928                 link_info->advertising = link_info->support_auto_speeds;
5929                 if (link_info->autoneg & BNXT_AUTONEG_SPEED)
5930                         bnxt_hwrm_set_link_setting(bp, true, false);
5931         }
5932         return 0;
5933 }
5934
5935 static void bnxt_get_port_module_status(struct bnxt *bp)
5936 {
5937         struct bnxt_link_info *link_info = &bp->link_info;
5938         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
5939         u8 module_status;
5940
5941         if (bnxt_update_link(bp, true))
5942                 return;
5943
5944         module_status = link_info->module_status;
5945         switch (module_status) {
5946         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
5947         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
5948         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
5949                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
5950                             bp->pf.port_id);
5951                 if (bp->hwrm_spec_code >= 0x10201) {
5952                         netdev_warn(bp->dev, "Module part number %s\n",
5953                                     resp->phy_vendor_partnumber);
5954                 }
5955                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
5956                         netdev_warn(bp->dev, "TX is disabled\n");
5957                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
5958                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
5959         }
5960 }
5961
5962 static void
5963 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
5964 {
5965         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
5966                 if (bp->hwrm_spec_code >= 0x10201)
5967                         req->auto_pause =
5968                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
5969                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
5970                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
5971                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
5972                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
5973                 req->enables |=
5974                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
5975         } else {
5976                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
5977                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
5978                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
5979                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
5980                 req->enables |=
5981                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
5982                 if (bp->hwrm_spec_code >= 0x10201) {
5983                         req->auto_pause = req->force_pause;
5984                         req->enables |= cpu_to_le32(
5985                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
5986                 }
5987         }
5988 }
5989
5990 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
5991                                       struct hwrm_port_phy_cfg_input *req)
5992 {
5993         u8 autoneg = bp->link_info.autoneg;
5994         u16 fw_link_speed = bp->link_info.req_link_speed;
5995         u16 advertising = bp->link_info.advertising;
5996
5997         if (autoneg & BNXT_AUTONEG_SPEED) {
5998                 req->auto_mode |=
5999                         PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
6000
6001                 req->enables |= cpu_to_le32(
6002                         PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
6003                 req->auto_link_speed_mask = cpu_to_le16(advertising);
6004
6005                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
6006                 req->flags |=
6007                         cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
6008         } else {
6009                 req->force_link_speed = cpu_to_le16(fw_link_speed);
6010                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
6011         }
6012
6013         /* tell chimp that the setting takes effect immediately */
6014         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
6015 }
6016
6017 int bnxt_hwrm_set_pause(struct bnxt *bp)
6018 {
6019         struct hwrm_port_phy_cfg_input req = {0};
6020         int rc;
6021
6022         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
6023         bnxt_hwrm_set_pause_common(bp, &req);
6024
6025         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
6026             bp->link_info.force_link_chng)
6027                 bnxt_hwrm_set_link_common(bp, &req);
6028
6029         mutex_lock(&bp->hwrm_cmd_lock);
6030         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6031         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
6032                 /* since changing of pause setting doesn't trigger any link
6033                  * change event, the driver needs to update the current pause
6034                  * result upon successfully return of the phy_cfg command
6035                  */
6036                 bp->link_info.pause =
6037                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
6038                 bp->link_info.auto_pause_setting = 0;
6039                 if (!bp->link_info.force_link_chng)
6040                         bnxt_report_link(bp);
6041         }
6042         bp->link_info.force_link_chng = false;
6043         mutex_unlock(&bp->hwrm_cmd_lock);
6044         return rc;
6045 }
6046
6047 static void bnxt_hwrm_set_eee(struct bnxt *bp,
6048                               struct hwrm_port_phy_cfg_input *req)
6049 {
6050         struct ethtool_eee *eee = &bp->eee;
6051
6052         if (eee->eee_enabled) {
6053                 u16 eee_speeds;
6054                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
6055
6056                 if (eee->tx_lpi_enabled)
6057                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
6058                 else
6059                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
6060
6061                 req->flags |= cpu_to_le32(flags);
6062                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
6063                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
6064                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
6065         } else {
6066                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
6067         }
6068 }
6069
6070 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
6071 {
6072         struct hwrm_port_phy_cfg_input req = {0};
6073
6074         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
6075         if (set_pause)
6076                 bnxt_hwrm_set_pause_common(bp, &req);
6077
6078         bnxt_hwrm_set_link_common(bp, &req);
6079
6080         if (set_eee)
6081                 bnxt_hwrm_set_eee(bp, &req);
6082         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6083 }
6084
6085 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
6086 {
6087         struct hwrm_port_phy_cfg_input req = {0};
6088
6089         if (!BNXT_SINGLE_PF(bp))
6090                 return 0;
6091
6092         if (pci_num_vf(bp->pdev))
6093                 return 0;
6094
6095         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
6096         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
6097         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6098 }
6099
6100 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
6101 {
6102         struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6103         struct hwrm_port_led_qcaps_input req = {0};
6104         struct bnxt_pf_info *pf = &bp->pf;
6105         int rc;
6106
6107         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
6108                 return 0;
6109
6110         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
6111         req.port_id = cpu_to_le16(pf->port_id);
6112         mutex_lock(&bp->hwrm_cmd_lock);
6113         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6114         if (rc) {
6115                 mutex_unlock(&bp->hwrm_cmd_lock);
6116                 return rc;
6117         }
6118         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
6119                 int i;
6120
6121                 bp->num_leds = resp->num_leds;
6122                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
6123                                                  bp->num_leds);
6124                 for (i = 0; i < bp->num_leds; i++) {
6125                         struct bnxt_led_info *led = &bp->leds[i];
6126                         __le16 caps = led->led_state_caps;
6127
6128                         if (!led->led_group_id ||
6129                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
6130                                 bp->num_leds = 0;
6131                                 break;
6132                         }
6133                 }
6134         }
6135         mutex_unlock(&bp->hwrm_cmd_lock);
6136         return 0;
6137 }
6138
6139 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
6140 {
6141         struct hwrm_wol_filter_alloc_input req = {0};
6142         struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6143         int rc;
6144
6145         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
6146         req.port_id = cpu_to_le16(bp->pf.port_id);
6147         req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
6148         req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
6149         memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
6150         mutex_lock(&bp->hwrm_cmd_lock);
6151         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6152         if (!rc)
6153                 bp->wol_filter_id = resp->wol_filter_id;
6154         mutex_unlock(&bp->hwrm_cmd_lock);
6155         return rc;
6156 }
6157
6158 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
6159 {
6160         struct hwrm_wol_filter_free_input req = {0};
6161         int rc;
6162
6163         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
6164         req.port_id = cpu_to_le16(bp->pf.port_id);
6165         req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
6166         req.wol_filter_id = bp->wol_filter_id;
6167         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6168         return rc;
6169 }
6170
6171 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
6172 {
6173         struct hwrm_wol_filter_qcfg_input req = {0};
6174         struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6175         u16 next_handle = 0;
6176         int rc;
6177
6178         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
6179         req.port_id = cpu_to_le16(bp->pf.port_id);
6180         req.handle = cpu_to_le16(handle);
6181         mutex_lock(&bp->hwrm_cmd_lock);
6182         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6183         if (!rc) {
6184                 next_handle = le16_to_cpu(resp->next_handle);
6185                 if (next_handle != 0) {
6186                         if (resp->wol_type ==
6187                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
6188                                 bp->wol = 1;
6189                                 bp->wol_filter_id = resp->wol_filter_id;
6190                         }
6191                 }
6192         }
6193         mutex_unlock(&bp->hwrm_cmd_lock);
6194         return next_handle;
6195 }
6196
6197 static void bnxt_get_wol_settings(struct bnxt *bp)
6198 {
6199         u16 handle = 0;
6200
6201         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
6202                 return;
6203
6204         do {
6205                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
6206         } while (handle && handle != 0xffff);
6207 }
6208
6209 static bool bnxt_eee_config_ok(struct bnxt *bp)
6210 {
6211         struct ethtool_eee *eee = &bp->eee;
6212         struct bnxt_link_info *link_info = &bp->link_info;
6213
6214         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
6215                 return true;
6216
6217         if (eee->eee_enabled) {
6218                 u32 advertising =
6219                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
6220
6221                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
6222                         eee->eee_enabled = 0;
6223                         return false;
6224                 }
6225                 if (eee->advertised & ~advertising) {
6226                         eee->advertised = advertising & eee->supported;
6227                         return false;
6228                 }
6229         }
6230         return true;
6231 }
6232
6233 static int bnxt_update_phy_setting(struct bnxt *bp)
6234 {
6235         int rc;
6236         bool update_link = false;
6237         bool update_pause = false;
6238         bool update_eee = false;
6239         struct bnxt_link_info *link_info = &bp->link_info;
6240
6241         rc = bnxt_update_link(bp, true);
6242         if (rc) {
6243                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
6244                            rc);
6245                 return rc;
6246         }
6247         if (!BNXT_SINGLE_PF(bp))
6248                 return 0;
6249
6250         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
6251             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
6252             link_info->req_flow_ctrl)
6253                 update_pause = true;
6254         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
6255             link_info->force_pause_setting != link_info->req_flow_ctrl)
6256                 update_pause = true;
6257         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
6258                 if (BNXT_AUTO_MODE(link_info->auto_mode))
6259                         update_link = true;
6260                 if (link_info->req_link_speed != link_info->force_link_speed)
6261                         update_link = true;
6262                 if (link_info->req_duplex != link_info->duplex_setting)
6263                         update_link = true;
6264         } else {
6265                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
6266                         update_link = true;
6267                 if (link_info->advertising != link_info->auto_link_speeds)
6268                         update_link = true;
6269         }
6270
6271         /* The last close may have shutdown the link, so need to call
6272          * PHY_CFG to bring it back up.
6273          */
6274         if (!netif_carrier_ok(bp->dev))
6275                 update_link = true;
6276
6277         if (!bnxt_eee_config_ok(bp))
6278                 update_eee = true;
6279
6280         if (update_link)
6281                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
6282         else if (update_pause)
6283                 rc = bnxt_hwrm_set_pause(bp);
6284         if (rc) {
6285                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
6286                            rc);
6287                 return rc;
6288         }
6289
6290         return rc;
6291 }
6292
6293 /* Common routine to pre-map certain register block to different GRC window.
6294  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
6295  * in PF and 3 windows in VF that can be customized to map in different
6296  * register blocks.
6297  */
6298 static void bnxt_preset_reg_win(struct bnxt *bp)
6299 {
6300         if (BNXT_PF(bp)) {
6301                 /* CAG registers map to GRC window #4 */
6302                 writel(BNXT_CAG_REG_BASE,
6303                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
6304         }
6305 }
6306
6307 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
6308 {
6309         int rc = 0;
6310
6311         bnxt_preset_reg_win(bp);
6312         netif_carrier_off(bp->dev);
6313         if (irq_re_init) {
6314                 rc = bnxt_setup_int_mode(bp);
6315                 if (rc) {
6316                         netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
6317                                    rc);
6318                         return rc;
6319                 }
6320         }
6321         if ((bp->flags & BNXT_FLAG_RFS) &&
6322             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
6323                 /* disable RFS if falling back to INTA */
6324                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
6325                 bp->flags &= ~BNXT_FLAG_RFS;
6326         }
6327
6328         rc = bnxt_alloc_mem(bp, irq_re_init);
6329         if (rc) {
6330                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
6331                 goto open_err_free_mem;
6332         }
6333
6334         if (irq_re_init) {
6335                 bnxt_init_napi(bp);
6336                 rc = bnxt_request_irq(bp);
6337                 if (rc) {
6338                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
6339                         goto open_err;
6340                 }
6341         }
6342
6343         bnxt_enable_napi(bp);
6344
6345         rc = bnxt_init_nic(bp, irq_re_init);
6346         if (rc) {
6347                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
6348                 goto open_err;
6349         }
6350
6351         if (link_re_init) {
6352                 mutex_lock(&bp->link_lock);
6353                 rc = bnxt_update_phy_setting(bp);
6354                 mutex_unlock(&bp->link_lock);
6355                 if (rc)
6356                         netdev_warn(bp->dev, "failed to update phy settings\n");
6357         }
6358
6359         if (irq_re_init)
6360                 udp_tunnel_get_rx_info(bp->dev);
6361
6362         set_bit(BNXT_STATE_OPEN, &bp->state);
6363         bnxt_enable_int(bp);
6364         /* Enable TX queues */
6365         bnxt_tx_enable(bp);
6366         mod_timer(&bp->timer, jiffies + bp->current_interval);
6367         /* Poll link status and check for SFP+ module status */
6368         bnxt_get_port_module_status(bp);
6369
6370         /* VF-reps may need to be re-opened after the PF is re-opened */
6371         if (BNXT_PF(bp))
6372                 bnxt_vf_reps_open(bp);
6373         return 0;
6374
6375 open_err:
6376         bnxt_disable_napi(bp);
6377         bnxt_del_napi(bp);
6378
6379 open_err_free_mem:
6380         bnxt_free_skbs(bp);
6381         bnxt_free_irq(bp);
6382         bnxt_free_mem(bp, true);
6383         return rc;
6384 }
6385
6386 /* rtnl_lock held */
6387 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
6388 {
6389         int rc = 0;
6390
6391         rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
6392         if (rc) {
6393                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
6394                 dev_close(bp->dev);
6395         }
6396         return rc;
6397 }
6398
6399 /* rtnl_lock held, open the NIC half way by allocating all resources, but
6400  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
6401  * self tests.
6402  */
6403 int bnxt_half_open_nic(struct bnxt *bp)
6404 {
6405         int rc = 0;
6406
6407         rc = bnxt_alloc_mem(bp, false);
6408         if (rc) {
6409                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
6410                 goto half_open_err;
6411         }
6412         rc = bnxt_init_nic(bp, false);
6413         if (rc) {
6414                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
6415                 goto half_open_err;
6416         }
6417         return 0;
6418
6419 half_open_err:
6420         bnxt_free_skbs(bp);
6421         bnxt_free_mem(bp, false);
6422         dev_close(bp->dev);
6423         return rc;
6424 }
6425
6426 /* rtnl_lock held, this call can only be made after a previous successful
6427  * call to bnxt_half_open_nic().
6428  */
6429 void bnxt_half_close_nic(struct bnxt *bp)
6430 {
6431         bnxt_hwrm_resource_free(bp, false, false);
6432         bnxt_free_skbs(bp);
6433         bnxt_free_mem(bp, false);
6434 }
6435
6436 static int bnxt_open(struct net_device *dev)
6437 {
6438         struct bnxt *bp = netdev_priv(dev);
6439
6440         return __bnxt_open_nic(bp, true, true);
6441 }
6442
6443 static bool bnxt_drv_busy(struct bnxt *bp)
6444 {
6445         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
6446                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
6447 }
6448
6449 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
6450 {
6451         int rc = 0;
6452
6453 #ifdef CONFIG_BNXT_SRIOV
6454         if (bp->sriov_cfg) {
6455                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
6456                                                       !bp->sriov_cfg,
6457                                                       BNXT_SRIOV_CFG_WAIT_TMO);
6458                 if (rc)
6459                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
6460         }
6461
6462         /* Close the VF-reps before closing PF */
6463         if (BNXT_PF(bp))
6464                 bnxt_vf_reps_close(bp);
6465 #endif
6466         /* Change device state to avoid TX queue wake up's */
6467         bnxt_tx_disable(bp);
6468
6469         clear_bit(BNXT_STATE_OPEN, &bp->state);
6470         smp_mb__after_atomic();
6471         while (bnxt_drv_busy(bp))
6472                 msleep(20);
6473
6474         /* Flush rings and and disable interrupts */
6475         bnxt_shutdown_nic(bp, irq_re_init);
6476
6477         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
6478
6479         bnxt_disable_napi(bp);
6480         del_timer_sync(&bp->timer);
6481         bnxt_free_skbs(bp);
6482
6483         if (irq_re_init) {
6484                 bnxt_free_irq(bp);
6485                 bnxt_del_napi(bp);
6486         }
6487         bnxt_free_mem(bp, irq_re_init);
6488         return rc;
6489 }
6490
6491 static int bnxt_close(struct net_device *dev)
6492 {
6493         struct bnxt *bp = netdev_priv(dev);
6494
6495         bnxt_close_nic(bp, true, true);
6496         bnxt_hwrm_shutdown_link(bp);
6497         return 0;
6498 }
6499
6500 /* rtnl_lock held */
6501 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
6502 {
6503         switch (cmd) {
6504         case SIOCGMIIPHY:
6505                 /* fallthru */
6506         case SIOCGMIIREG: {
6507                 if (!netif_running(dev))
6508                         return -EAGAIN;
6509
6510                 return 0;
6511         }
6512
6513         case SIOCSMIIREG:
6514                 if (!netif_running(dev))
6515                         return -EAGAIN;
6516
6517                 return 0;
6518
6519         default:
6520                 /* do nothing */
6521                 break;
6522         }
6523         return -EOPNOTSUPP;
6524 }
6525
6526 static void
6527 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
6528 {
6529         u32 i;
6530         struct bnxt *bp = netdev_priv(dev);
6531
6532         set_bit(BNXT_STATE_READ_STATS, &bp->state);
6533         /* Make sure bnxt_close_nic() sees that we are reading stats before
6534          * we check the BNXT_STATE_OPEN flag.
6535          */
6536         smp_mb__after_atomic();
6537         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
6538                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
6539                 return;
6540         }
6541
6542         /* TODO check if we need to synchronize with bnxt_close path */
6543         for (i = 0; i < bp->cp_nr_rings; i++) {
6544                 struct bnxt_napi *bnapi = bp->bnapi[i];
6545                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6546                 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
6547
6548                 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
6549                 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
6550                 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
6551
6552                 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
6553                 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
6554                 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
6555
6556                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
6557                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
6558                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
6559
6560                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
6561                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
6562                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
6563
6564                 stats->rx_missed_errors +=
6565                         le64_to_cpu(hw_stats->rx_discard_pkts);
6566
6567                 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
6568
6569                 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
6570         }
6571
6572         if (bp->flags & BNXT_FLAG_PORT_STATS) {
6573                 struct rx_port_stats *rx = bp->hw_rx_port_stats;
6574                 struct tx_port_stats *tx = bp->hw_tx_port_stats;
6575
6576                 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
6577                 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
6578                 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
6579                                           le64_to_cpu(rx->rx_ovrsz_frames) +
6580                                           le64_to_cpu(rx->rx_runt_frames);
6581                 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
6582                                    le64_to_cpu(rx->rx_jbr_frames);
6583                 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
6584                 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
6585                 stats->tx_errors = le64_to_cpu(tx->tx_err);
6586         }
6587         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
6588 }
6589
6590 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
6591 {
6592         struct net_device *dev = bp->dev;
6593         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6594         struct netdev_hw_addr *ha;
6595         u8 *haddr;
6596         int mc_count = 0;
6597         bool update = false;
6598         int off = 0;
6599
6600         netdev_for_each_mc_addr(ha, dev) {
6601                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
6602                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
6603                         vnic->mc_list_count = 0;
6604                         return false;
6605                 }
6606                 haddr = ha->addr;
6607                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
6608                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
6609                         update = true;
6610                 }
6611                 off += ETH_ALEN;
6612                 mc_count++;
6613         }
6614         if (mc_count)
6615                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
6616
6617         if (mc_count != vnic->mc_list_count) {
6618                 vnic->mc_list_count = mc_count;
6619                 update = true;
6620         }
6621         return update;
6622 }
6623
6624 static bool bnxt_uc_list_updated(struct bnxt *bp)
6625 {
6626         struct net_device *dev = bp->dev;
6627         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6628         struct netdev_hw_addr *ha;
6629         int off = 0;
6630
6631         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
6632                 return true;
6633
6634         netdev_for_each_uc_addr(ha, dev) {
6635                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
6636                         return true;
6637
6638                 off += ETH_ALEN;
6639         }
6640         return false;
6641 }
6642
6643 static void bnxt_set_rx_mode(struct net_device *dev)
6644 {
6645         struct bnxt *bp = netdev_priv(dev);
6646         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6647         u32 mask = vnic->rx_mask;
6648         bool mc_update = false;
6649         bool uc_update;
6650
6651         if (!netif_running(dev))
6652                 return;
6653
6654         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
6655                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
6656                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST);
6657
6658         if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
6659                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
6660
6661         uc_update = bnxt_uc_list_updated(bp);
6662
6663         if (dev->flags & IFF_ALLMULTI) {
6664                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
6665                 vnic->mc_list_count = 0;
6666         } else {
6667                 mc_update = bnxt_mc_list_updated(bp, &mask);
6668         }
6669
6670         if (mask != vnic->rx_mask || uc_update || mc_update) {
6671                 vnic->rx_mask = mask;
6672
6673                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
6674                 bnxt_queue_sp_work(bp);
6675         }
6676 }
6677
6678 static int bnxt_cfg_rx_mode(struct bnxt *bp)
6679 {
6680         struct net_device *dev = bp->dev;
6681         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6682         struct netdev_hw_addr *ha;
6683         int i, off = 0, rc;
6684         bool uc_update;
6685
6686         netif_addr_lock_bh(dev);
6687         uc_update = bnxt_uc_list_updated(bp);
6688         netif_addr_unlock_bh(dev);
6689
6690         if (!uc_update)
6691                 goto skip_uc;
6692
6693         mutex_lock(&bp->hwrm_cmd_lock);
6694         for (i = 1; i < vnic->uc_filter_count; i++) {
6695                 struct hwrm_cfa_l2_filter_free_input req = {0};
6696
6697                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
6698                                        -1);
6699
6700                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
6701
6702                 rc = _hwrm_send_message(bp, &req, sizeof(req),
6703                                         HWRM_CMD_TIMEOUT);
6704         }
6705         mutex_unlock(&bp->hwrm_cmd_lock);
6706
6707         vnic->uc_filter_count = 1;
6708
6709         netif_addr_lock_bh(dev);
6710         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
6711                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
6712         } else {
6713                 netdev_for_each_uc_addr(ha, dev) {
6714                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
6715                         off += ETH_ALEN;
6716                         vnic->uc_filter_count++;
6717                 }
6718         }
6719         netif_addr_unlock_bh(dev);
6720
6721         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
6722                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
6723                 if (rc) {
6724                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
6725                                    rc);
6726                         vnic->uc_filter_count = i;
6727                         return rc;
6728                 }
6729         }
6730
6731 skip_uc:
6732         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
6733         if (rc)
6734                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n",
6735                            rc);
6736
6737         return rc;
6738 }
6739
6740 /* If the chip and firmware supports RFS */
6741 static bool bnxt_rfs_supported(struct bnxt *bp)
6742 {
6743         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
6744                 return true;
6745         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
6746                 return true;
6747         return false;
6748 }
6749
6750 /* If runtime conditions support RFS */
6751 static bool bnxt_rfs_capable(struct bnxt *bp)
6752 {
6753 #ifdef CONFIG_RFS_ACCEL
6754         int vnics, max_vnics, max_rss_ctxs;
6755
6756         if (!(bp->flags & BNXT_FLAG_MSIX_CAP))
6757                 return false;
6758
6759         vnics = 1 + bp->rx_nr_rings;
6760         max_vnics = bnxt_get_max_func_vnics(bp);
6761         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
6762
6763         /* RSS contexts not a limiting factor */
6764         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
6765                 max_rss_ctxs = max_vnics;
6766         if (vnics > max_vnics || vnics > max_rss_ctxs) {
6767                 netdev_warn(bp->dev,
6768                             "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
6769                             min(max_rss_ctxs - 1, max_vnics - 1));
6770                 return false;
6771         }
6772
6773         return true;
6774 #else
6775         return false;
6776 #endif
6777 }
6778
6779 static netdev_features_t bnxt_fix_features(struct net_device *dev,
6780                                            netdev_features_t features)
6781 {
6782         struct bnxt *bp = netdev_priv(dev);
6783
6784         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
6785                 features &= ~NETIF_F_NTUPLE;
6786
6787         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
6788          * turned on or off together.
6789          */
6790         if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) !=
6791             (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) {
6792                 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
6793                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
6794                                       NETIF_F_HW_VLAN_STAG_RX);
6795                 else
6796                         features |= NETIF_F_HW_VLAN_CTAG_RX |
6797                                     NETIF_F_HW_VLAN_STAG_RX;
6798         }
6799 #ifdef CONFIG_BNXT_SRIOV
6800         if (BNXT_VF(bp)) {
6801                 if (bp->vf.vlan) {
6802                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
6803                                       NETIF_F_HW_VLAN_STAG_RX);
6804                 }
6805         }
6806 #endif
6807         return features;
6808 }
6809
6810 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
6811 {
6812         struct bnxt *bp = netdev_priv(dev);
6813         u32 flags = bp->flags;
6814         u32 changes;
6815         int rc = 0;
6816         bool re_init = false;
6817         bool update_tpa = false;
6818
6819         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
6820         if ((features & NETIF_F_GRO) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
6821                 flags |= BNXT_FLAG_GRO;
6822         if (features & NETIF_F_LRO)
6823                 flags |= BNXT_FLAG_LRO;
6824
6825         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
6826                 flags &= ~BNXT_FLAG_TPA;
6827
6828         if (features & NETIF_F_HW_VLAN_CTAG_RX)
6829                 flags |= BNXT_FLAG_STRIP_VLAN;
6830
6831         if (features & NETIF_F_NTUPLE)
6832                 flags |= BNXT_FLAG_RFS;
6833
6834         changes = flags ^ bp->flags;
6835         if (changes & BNXT_FLAG_TPA) {
6836                 update_tpa = true;
6837                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
6838                     (flags & BNXT_FLAG_TPA) == 0)
6839                         re_init = true;
6840         }
6841
6842         if (changes & ~BNXT_FLAG_TPA)
6843                 re_init = true;
6844
6845         if (flags != bp->flags) {
6846                 u32 old_flags = bp->flags;
6847
6848                 bp->flags = flags;
6849
6850                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
6851                         if (update_tpa)
6852                                 bnxt_set_ring_params(bp);
6853                         return rc;
6854                 }
6855
6856                 if (re_init) {
6857                         bnxt_close_nic(bp, false, false);
6858                         if (update_tpa)
6859                                 bnxt_set_ring_params(bp);
6860
6861                         return bnxt_open_nic(bp, false, false);
6862                 }
6863                 if (update_tpa) {
6864                         rc = bnxt_set_tpa(bp,
6865                                           (flags & BNXT_FLAG_TPA) ?
6866                                           true : false);
6867                         if (rc)
6868                                 bp->flags = old_flags;
6869                 }
6870         }
6871         return rc;
6872 }
6873
6874 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
6875 {
6876         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
6877         int i = bnapi->index;
6878
6879         if (!txr)
6880                 return;
6881
6882         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
6883                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
6884                     txr->tx_cons);
6885 }
6886
6887 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
6888 {
6889         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
6890         int i = bnapi->index;
6891
6892         if (!rxr)
6893                 return;
6894
6895         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
6896                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
6897                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
6898                     rxr->rx_sw_agg_prod);
6899 }
6900
6901 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
6902 {
6903         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6904         int i = bnapi->index;
6905
6906         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
6907                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
6908 }
6909
6910 static void bnxt_dbg_dump_states(struct bnxt *bp)
6911 {
6912         int i;
6913         struct bnxt_napi *bnapi;
6914
6915         for (i = 0; i < bp->cp_nr_rings; i++) {
6916                 bnapi = bp->bnapi[i];
6917                 if (netif_msg_drv(bp)) {
6918                         bnxt_dump_tx_sw_state(bnapi);
6919                         bnxt_dump_rx_sw_state(bnapi);
6920                         bnxt_dump_cp_sw_state(bnapi);
6921                 }
6922         }
6923 }
6924
6925 static void bnxt_reset_task(struct bnxt *bp, bool silent)
6926 {
6927         if (!silent)
6928                 bnxt_dbg_dump_states(bp);
6929         if (netif_running(bp->dev)) {
6930                 int rc;
6931
6932                 if (!silent)
6933                         bnxt_ulp_stop(bp);
6934                 bnxt_close_nic(bp, false, false);
6935                 rc = bnxt_open_nic(bp, false, false);
6936                 if (!silent && !rc)
6937                         bnxt_ulp_start(bp);
6938         }
6939 }
6940
6941 static void bnxt_tx_timeout(struct net_device *dev)
6942 {
6943         struct bnxt *bp = netdev_priv(dev);
6944
6945         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
6946         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
6947         bnxt_queue_sp_work(bp);
6948 }
6949
6950 #ifdef CONFIG_NET_POLL_CONTROLLER
6951 static void bnxt_poll_controller(struct net_device *dev)
6952 {
6953         struct bnxt *bp = netdev_priv(dev);
6954         int i;
6955
6956         /* Only process tx rings/combined rings in netpoll mode. */
6957         for (i = 0; i < bp->tx_nr_rings; i++) {
6958                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
6959
6960                 napi_schedule(&txr->bnapi->napi);
6961         }
6962 }
6963 #endif
6964
6965 static void bnxt_timer(struct timer_list *t)
6966 {
6967         struct bnxt *bp = from_timer(bp, t, timer);
6968         struct net_device *dev = bp->dev;
6969
6970         if (!netif_running(dev))
6971                 return;
6972
6973         if (atomic_read(&bp->intr_sem) != 0)
6974                 goto bnxt_restart_timer;
6975
6976         if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) &&
6977             bp->stats_coal_ticks) {
6978                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
6979                 bnxt_queue_sp_work(bp);
6980         }
6981
6982         if (bnxt_tc_flower_enabled(bp)) {
6983                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
6984                 bnxt_queue_sp_work(bp);
6985         }
6986 bnxt_restart_timer:
6987         mod_timer(&bp->timer, jiffies + bp->current_interval);
6988 }
6989
6990 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
6991 {
6992         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
6993          * set.  If the device is being closed, bnxt_close() may be holding
6994          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
6995          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
6996          */
6997         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
6998         rtnl_lock();
6999 }
7000
7001 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
7002 {
7003         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7004         rtnl_unlock();
7005 }
7006
7007 /* Only called from bnxt_sp_task() */
7008 static void bnxt_reset(struct bnxt *bp, bool silent)
7009 {
7010         bnxt_rtnl_lock_sp(bp);
7011         if (test_bit(BNXT_STATE_OPEN, &bp->state))
7012                 bnxt_reset_task(bp, silent);
7013         bnxt_rtnl_unlock_sp(bp);
7014 }
7015
7016 static void bnxt_cfg_ntp_filters(struct bnxt *);
7017
7018 static void bnxt_sp_task(struct work_struct *work)
7019 {
7020         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
7021
7022         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7023         smp_mb__after_atomic();
7024         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
7025                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7026                 return;
7027         }
7028
7029         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
7030                 bnxt_cfg_rx_mode(bp);
7031
7032         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
7033                 bnxt_cfg_ntp_filters(bp);
7034         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
7035                 bnxt_hwrm_exec_fwd_req(bp);
7036         if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
7037                 bnxt_hwrm_tunnel_dst_port_alloc(
7038                         bp, bp->vxlan_port,
7039                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7040         }
7041         if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
7042                 bnxt_hwrm_tunnel_dst_port_free(
7043                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7044         }
7045         if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) {
7046                 bnxt_hwrm_tunnel_dst_port_alloc(
7047                         bp, bp->nge_port,
7048                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7049         }
7050         if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) {
7051                 bnxt_hwrm_tunnel_dst_port_free(
7052                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7053         }
7054         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event))
7055                 bnxt_hwrm_port_qstats(bp);
7056
7057         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
7058                 int rc;
7059
7060                 mutex_lock(&bp->link_lock);
7061                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
7062                                        &bp->sp_event))
7063                         bnxt_hwrm_phy_qcaps(bp);
7064
7065                 rc = bnxt_update_link(bp, true);
7066                 mutex_unlock(&bp->link_lock);
7067                 if (rc)
7068                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
7069                                    rc);
7070         }
7071         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
7072                 mutex_lock(&bp->link_lock);
7073                 bnxt_get_port_module_status(bp);
7074                 mutex_unlock(&bp->link_lock);
7075         }
7076
7077         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
7078                 bnxt_tc_flow_stats_work(bp);
7079
7080         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
7081          * must be the last functions to be called before exiting.
7082          */
7083         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
7084                 bnxt_reset(bp, false);
7085
7086         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
7087                 bnxt_reset(bp, true);
7088
7089         smp_mb__before_atomic();
7090         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7091 }
7092
7093 /* Under rtnl_lock */
7094 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
7095                      int tx_xdp)
7096 {
7097         int max_rx, max_tx, tx_sets = 1;
7098         int tx_rings_needed;
7099         int rc;
7100
7101         if (tcs)
7102                 tx_sets = tcs;
7103
7104         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
7105         if (rc)
7106                 return rc;
7107
7108         if (max_rx < rx)
7109                 return -ENOMEM;
7110
7111         tx_rings_needed = tx * tx_sets + tx_xdp;
7112         if (max_tx < tx_rings_needed)
7113                 return -ENOMEM;
7114
7115         return bnxt_hwrm_check_tx_rings(bp, tx_rings_needed);
7116 }
7117
7118 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
7119 {
7120         if (bp->bar2) {
7121                 pci_iounmap(pdev, bp->bar2);
7122                 bp->bar2 = NULL;
7123         }
7124
7125         if (bp->bar1) {
7126                 pci_iounmap(pdev, bp->bar1);
7127                 bp->bar1 = NULL;
7128         }
7129
7130         if (bp->bar0) {
7131                 pci_iounmap(pdev, bp->bar0);
7132                 bp->bar0 = NULL;
7133         }
7134 }
7135
7136 static void bnxt_cleanup_pci(struct bnxt *bp)
7137 {
7138         bnxt_unmap_bars(bp, bp->pdev);
7139         pci_release_regions(bp->pdev);
7140         pci_disable_device(bp->pdev);
7141 }
7142
7143 static void bnxt_init_dflt_coal(struct bnxt *bp)
7144 {
7145         struct bnxt_coal *coal;
7146
7147         /* Tick values in micro seconds.
7148          * 1 coal_buf x bufs_per_record = 1 completion record.
7149          */
7150         coal = &bp->rx_coal;
7151         coal->coal_ticks = 14;
7152         coal->coal_bufs = 30;
7153         coal->coal_ticks_irq = 1;
7154         coal->coal_bufs_irq = 2;
7155         coal->idle_thresh = 25;
7156         coal->bufs_per_record = 2;
7157         coal->budget = 64;              /* NAPI budget */
7158
7159         coal = &bp->tx_coal;
7160         coal->coal_ticks = 28;
7161         coal->coal_bufs = 30;
7162         coal->coal_ticks_irq = 2;
7163         coal->coal_bufs_irq = 2;
7164         coal->bufs_per_record = 1;
7165
7166         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
7167 }
7168
7169 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
7170 {
7171         int rc;
7172         struct bnxt *bp = netdev_priv(dev);
7173
7174         SET_NETDEV_DEV(dev, &pdev->dev);
7175
7176         /* enable device (incl. PCI PM wakeup), and bus-mastering */
7177         rc = pci_enable_device(pdev);
7178         if (rc) {
7179                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
7180                 goto init_err;
7181         }
7182
7183         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
7184                 dev_err(&pdev->dev,
7185                         "Cannot find PCI device base address, aborting\n");
7186                 rc = -ENODEV;
7187                 goto init_err_disable;
7188         }
7189
7190         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
7191         if (rc) {
7192                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
7193                 goto init_err_disable;
7194         }
7195
7196         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
7197             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
7198                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
7199                 goto init_err_disable;
7200         }
7201
7202         pci_set_master(pdev);
7203
7204         bp->dev = dev;
7205         bp->pdev = pdev;
7206
7207         bp->bar0 = pci_ioremap_bar(pdev, 0);
7208         if (!bp->bar0) {
7209                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
7210                 rc = -ENOMEM;
7211                 goto init_err_release;
7212         }
7213
7214         bp->bar1 = pci_ioremap_bar(pdev, 2);
7215         if (!bp->bar1) {
7216                 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
7217                 rc = -ENOMEM;
7218                 goto init_err_release;
7219         }
7220
7221         bp->bar2 = pci_ioremap_bar(pdev, 4);
7222         if (!bp->bar2) {
7223                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
7224                 rc = -ENOMEM;
7225                 goto init_err_release;
7226         }
7227
7228         pci_enable_pcie_error_reporting(pdev);
7229
7230         INIT_WORK(&bp->sp_task, bnxt_sp_task);
7231
7232         spin_lock_init(&bp->ntp_fltr_lock);
7233
7234         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
7235         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
7236
7237         bnxt_init_dflt_coal(bp);
7238
7239         timer_setup(&bp->timer, bnxt_timer, 0);
7240         bp->current_interval = BNXT_TIMER_INTERVAL;
7241
7242         clear_bit(BNXT_STATE_OPEN, &bp->state);
7243         return 0;
7244
7245 init_err_release:
7246         bnxt_unmap_bars(bp, pdev);
7247         pci_release_regions(pdev);
7248
7249 init_err_disable:
7250         pci_disable_device(pdev);
7251
7252 init_err:
7253         return rc;
7254 }
7255
7256 /* rtnl_lock held */
7257 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
7258 {
7259         struct sockaddr *addr = p;
7260         struct bnxt *bp = netdev_priv(dev);
7261         int rc = 0;
7262
7263         if (!is_valid_ether_addr(addr->sa_data))
7264                 return -EADDRNOTAVAIL;
7265
7266         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
7267                 return 0;
7268
7269         rc = bnxt_approve_mac(bp, addr->sa_data);
7270         if (rc)
7271                 return rc;
7272
7273         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
7274         if (netif_running(dev)) {
7275                 bnxt_close_nic(bp, false, false);
7276                 rc = bnxt_open_nic(bp, false, false);
7277         }
7278
7279         return rc;
7280 }
7281
7282 /* rtnl_lock held */
7283 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
7284 {
7285         struct bnxt *bp = netdev_priv(dev);
7286
7287         if (netif_running(dev))
7288                 bnxt_close_nic(bp, false, false);
7289
7290         dev->mtu = new_mtu;
7291         bnxt_set_ring_params(bp);
7292
7293         if (netif_running(dev))
7294                 return bnxt_open_nic(bp, false, false);
7295
7296         return 0;
7297 }
7298
7299 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
7300 {
7301         struct bnxt *bp = netdev_priv(dev);
7302         bool sh = false;
7303         int rc;
7304
7305         if (tc > bp->max_tc) {
7306                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
7307                            tc, bp->max_tc);
7308                 return -EINVAL;
7309         }
7310
7311         if (netdev_get_num_tc(dev) == tc)
7312                 return 0;
7313
7314         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7315                 sh = true;
7316
7317         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
7318                               sh, tc, bp->tx_nr_rings_xdp);
7319         if (rc)
7320                 return rc;
7321
7322         /* Needs to close the device and do hw resource re-allocations */
7323         if (netif_running(bp->dev))
7324                 bnxt_close_nic(bp, true, false);
7325
7326         if (tc) {
7327                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
7328                 netdev_set_num_tc(dev, tc);
7329         } else {
7330                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
7331                 netdev_reset_tc(dev);
7332         }
7333         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
7334         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
7335                                bp->tx_nr_rings + bp->rx_nr_rings;
7336         bp->num_stat_ctxs = bp->cp_nr_rings;
7337
7338         if (netif_running(bp->dev))
7339                 return bnxt_open_nic(bp, true, false);
7340
7341         return 0;
7342 }
7343
7344 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
7345                                   void *cb_priv)
7346 {
7347         struct bnxt *bp = cb_priv;
7348
7349         if (!bnxt_tc_flower_enabled(bp) || !tc_can_offload(bp->dev))
7350                 return -EOPNOTSUPP;
7351
7352         switch (type) {
7353         case TC_SETUP_CLSFLOWER:
7354                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
7355         default:
7356                 return -EOPNOTSUPP;
7357         }
7358 }
7359
7360 static int bnxt_setup_tc_block(struct net_device *dev,
7361                                struct tc_block_offload *f)
7362 {
7363         struct bnxt *bp = netdev_priv(dev);
7364
7365         if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
7366                 return -EOPNOTSUPP;
7367
7368         switch (f->command) {
7369         case TC_BLOCK_BIND:
7370                 return tcf_block_cb_register(f->block, bnxt_setup_tc_block_cb,
7371                                              bp, bp);
7372         case TC_BLOCK_UNBIND:
7373                 tcf_block_cb_unregister(f->block, bnxt_setup_tc_block_cb, bp);
7374                 return 0;
7375         default:
7376                 return -EOPNOTSUPP;
7377         }
7378 }
7379
7380 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
7381                          void *type_data)
7382 {
7383         switch (type) {
7384         case TC_SETUP_BLOCK:
7385                 return bnxt_setup_tc_block(dev, type_data);
7386         case TC_SETUP_QDISC_MQPRIO: {
7387                 struct tc_mqprio_qopt *mqprio = type_data;
7388
7389                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
7390
7391                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
7392         }
7393         default:
7394                 return -EOPNOTSUPP;
7395         }
7396 }
7397
7398 #ifdef CONFIG_RFS_ACCEL
7399 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
7400                             struct bnxt_ntuple_filter *f2)
7401 {
7402         struct flow_keys *keys1 = &f1->fkeys;
7403         struct flow_keys *keys2 = &f2->fkeys;
7404
7405         if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
7406             keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
7407             keys1->ports.ports == keys2->ports.ports &&
7408             keys1->basic.ip_proto == keys2->basic.ip_proto &&
7409             keys1->basic.n_proto == keys2->basic.n_proto &&
7410             keys1->control.flags == keys2->control.flags &&
7411             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
7412             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
7413                 return true;
7414
7415         return false;
7416 }
7417
7418 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
7419                               u16 rxq_index, u32 flow_id)
7420 {
7421         struct bnxt *bp = netdev_priv(dev);
7422         struct bnxt_ntuple_filter *fltr, *new_fltr;
7423         struct flow_keys *fkeys;
7424         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
7425         int rc = 0, idx, bit_id, l2_idx = 0;
7426         struct hlist_head *head;
7427
7428         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
7429                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
7430                 int off = 0, j;
7431
7432                 netif_addr_lock_bh(dev);
7433                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
7434                         if (ether_addr_equal(eth->h_dest,
7435                                              vnic->uc_list + off)) {
7436                                 l2_idx = j + 1;
7437                                 break;
7438                         }
7439                 }
7440                 netif_addr_unlock_bh(dev);
7441                 if (!l2_idx)
7442                         return -EINVAL;
7443         }
7444         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
7445         if (!new_fltr)
7446                 return -ENOMEM;
7447
7448         fkeys = &new_fltr->fkeys;
7449         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
7450                 rc = -EPROTONOSUPPORT;
7451                 goto err_free;
7452         }
7453
7454         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
7455              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
7456             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
7457              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
7458                 rc = -EPROTONOSUPPORT;
7459                 goto err_free;
7460         }
7461         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
7462             bp->hwrm_spec_code < 0x10601) {
7463                 rc = -EPROTONOSUPPORT;
7464                 goto err_free;
7465         }
7466         if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) &&
7467             bp->hwrm_spec_code < 0x10601) {
7468                 rc = -EPROTONOSUPPORT;
7469                 goto err_free;
7470         }
7471
7472         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
7473         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
7474
7475         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
7476         head = &bp->ntp_fltr_hash_tbl[idx];
7477         rcu_read_lock();
7478         hlist_for_each_entry_rcu(fltr, head, hash) {
7479                 if (bnxt_fltr_match(fltr, new_fltr)) {
7480                         rcu_read_unlock();
7481                         rc = 0;
7482                         goto err_free;
7483                 }
7484         }
7485         rcu_read_unlock();
7486
7487         spin_lock_bh(&bp->ntp_fltr_lock);
7488         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
7489                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
7490         if (bit_id < 0) {
7491                 spin_unlock_bh(&bp->ntp_fltr_lock);
7492                 rc = -ENOMEM;
7493                 goto err_free;
7494         }
7495
7496         new_fltr->sw_id = (u16)bit_id;
7497         new_fltr->flow_id = flow_id;
7498         new_fltr->l2_fltr_idx = l2_idx;
7499         new_fltr->rxq = rxq_index;
7500         hlist_add_head_rcu(&new_fltr->hash, head);
7501         bp->ntp_fltr_count++;
7502         spin_unlock_bh(&bp->ntp_fltr_lock);
7503
7504         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
7505         bnxt_queue_sp_work(bp);
7506
7507         return new_fltr->sw_id;
7508
7509 err_free:
7510         kfree(new_fltr);
7511         return rc;
7512 }
7513
7514 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
7515 {
7516         int i;
7517
7518         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
7519                 struct hlist_head *head;
7520                 struct hlist_node *tmp;
7521                 struct bnxt_ntuple_filter *fltr;
7522                 int rc;
7523
7524                 head = &bp->ntp_fltr_hash_tbl[i];
7525                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
7526                         bool del = false;
7527
7528                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
7529                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
7530                                                         fltr->flow_id,
7531                                                         fltr->sw_id)) {
7532                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
7533                                                                          fltr);
7534                                         del = true;
7535                                 }
7536                         } else {
7537                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
7538                                                                        fltr);
7539                                 if (rc)
7540                                         del = true;
7541                                 else
7542                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
7543                         }
7544
7545                         if (del) {
7546                                 spin_lock_bh(&bp->ntp_fltr_lock);
7547                                 hlist_del_rcu(&fltr->hash);
7548                                 bp->ntp_fltr_count--;
7549                                 spin_unlock_bh(&bp->ntp_fltr_lock);
7550                                 synchronize_rcu();
7551                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
7552                                 kfree(fltr);
7553                         }
7554                 }
7555         }
7556         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
7557                 netdev_info(bp->dev, "Receive PF driver unload event!");
7558 }
7559
7560 #else
7561
7562 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
7563 {
7564 }
7565
7566 #endif /* CONFIG_RFS_ACCEL */
7567
7568 static void bnxt_udp_tunnel_add(struct net_device *dev,
7569                                 struct udp_tunnel_info *ti)
7570 {
7571         struct bnxt *bp = netdev_priv(dev);
7572
7573         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
7574                 return;
7575
7576         if (!netif_running(dev))
7577                 return;
7578
7579         switch (ti->type) {
7580         case UDP_TUNNEL_TYPE_VXLAN:
7581                 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port)
7582                         return;
7583
7584                 bp->vxlan_port_cnt++;
7585                 if (bp->vxlan_port_cnt == 1) {
7586                         bp->vxlan_port = ti->port;
7587                         set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
7588                         bnxt_queue_sp_work(bp);
7589                 }
7590                 break;
7591         case UDP_TUNNEL_TYPE_GENEVE:
7592                 if (bp->nge_port_cnt && bp->nge_port != ti->port)
7593                         return;
7594
7595                 bp->nge_port_cnt++;
7596                 if (bp->nge_port_cnt == 1) {
7597                         bp->nge_port = ti->port;
7598                         set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event);
7599                 }
7600                 break;
7601         default:
7602                 return;
7603         }
7604
7605         bnxt_queue_sp_work(bp);
7606 }
7607
7608 static void bnxt_udp_tunnel_del(struct net_device *dev,
7609                                 struct udp_tunnel_info *ti)
7610 {
7611         struct bnxt *bp = netdev_priv(dev);
7612
7613         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
7614                 return;
7615
7616         if (!netif_running(dev))
7617                 return;
7618
7619         switch (ti->type) {
7620         case UDP_TUNNEL_TYPE_VXLAN:
7621                 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port)
7622                         return;
7623                 bp->vxlan_port_cnt--;
7624
7625                 if (bp->vxlan_port_cnt != 0)
7626                         return;
7627
7628                 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
7629                 break;
7630         case UDP_TUNNEL_TYPE_GENEVE:
7631                 if (!bp->nge_port_cnt || bp->nge_port != ti->port)
7632                         return;
7633                 bp->nge_port_cnt--;
7634
7635                 if (bp->nge_port_cnt != 0)
7636                         return;
7637
7638                 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event);
7639                 break;
7640         default:
7641                 return;
7642         }
7643
7644         bnxt_queue_sp_work(bp);
7645 }
7646
7647 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
7648                                struct net_device *dev, u32 filter_mask,
7649                                int nlflags)
7650 {
7651         struct bnxt *bp = netdev_priv(dev);
7652
7653         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
7654                                        nlflags, filter_mask, NULL);
7655 }
7656
7657 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
7658                                u16 flags)
7659 {
7660         struct bnxt *bp = netdev_priv(dev);
7661         struct nlattr *attr, *br_spec;
7662         int rem, rc = 0;
7663
7664         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
7665                 return -EOPNOTSUPP;
7666
7667         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
7668         if (!br_spec)
7669                 return -EINVAL;
7670
7671         nla_for_each_nested(attr, br_spec, rem) {
7672                 u16 mode;
7673
7674                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
7675                         continue;
7676
7677                 if (nla_len(attr) < sizeof(mode))
7678                         return -EINVAL;
7679
7680                 mode = nla_get_u16(attr);
7681                 if (mode == bp->br_mode)
7682                         break;
7683
7684                 rc = bnxt_hwrm_set_br_mode(bp, mode);
7685                 if (!rc)
7686                         bp->br_mode = mode;
7687                 break;
7688         }
7689         return rc;
7690 }
7691
7692 static int bnxt_get_phys_port_name(struct net_device *dev, char *buf,
7693                                    size_t len)
7694 {
7695         struct bnxt *bp = netdev_priv(dev);
7696         int rc;
7697
7698         /* The PF and it's VF-reps only support the switchdev framework */
7699         if (!BNXT_PF(bp))
7700                 return -EOPNOTSUPP;
7701
7702         rc = snprintf(buf, len, "p%d", bp->pf.port_id);
7703
7704         if (rc >= len)
7705                 return -EOPNOTSUPP;
7706         return 0;
7707 }
7708
7709 int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr)
7710 {
7711         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
7712                 return -EOPNOTSUPP;
7713
7714         /* The PF and it's VF-reps only support the switchdev framework */
7715         if (!BNXT_PF(bp))
7716                 return -EOPNOTSUPP;
7717
7718         switch (attr->id) {
7719         case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
7720                 /* In SRIOV each PF-pool (PF + child VFs) serves as a
7721                  * switching domain, the PF's perm mac-addr can be used
7722                  * as the unique parent-id
7723                  */
7724                 attr->u.ppid.id_len = ETH_ALEN;
7725                 ether_addr_copy(attr->u.ppid.id, bp->pf.mac_addr);
7726                 break;
7727         default:
7728                 return -EOPNOTSUPP;
7729         }
7730         return 0;
7731 }
7732
7733 static int bnxt_swdev_port_attr_get(struct net_device *dev,
7734                                     struct switchdev_attr *attr)
7735 {
7736         return bnxt_port_attr_get(netdev_priv(dev), attr);
7737 }
7738
7739 static const struct switchdev_ops bnxt_switchdev_ops = {
7740         .switchdev_port_attr_get        = bnxt_swdev_port_attr_get
7741 };
7742
7743 static const struct net_device_ops bnxt_netdev_ops = {
7744         .ndo_open               = bnxt_open,
7745         .ndo_start_xmit         = bnxt_start_xmit,
7746         .ndo_stop               = bnxt_close,
7747         .ndo_get_stats64        = bnxt_get_stats64,
7748         .ndo_set_rx_mode        = bnxt_set_rx_mode,
7749         .ndo_do_ioctl           = bnxt_ioctl,
7750         .ndo_validate_addr      = eth_validate_addr,
7751         .ndo_set_mac_address    = bnxt_change_mac_addr,
7752         .ndo_change_mtu         = bnxt_change_mtu,
7753         .ndo_fix_features       = bnxt_fix_features,
7754         .ndo_set_features       = bnxt_set_features,
7755         .ndo_tx_timeout         = bnxt_tx_timeout,
7756 #ifdef CONFIG_BNXT_SRIOV
7757         .ndo_get_vf_config      = bnxt_get_vf_config,
7758         .ndo_set_vf_mac         = bnxt_set_vf_mac,
7759         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
7760         .ndo_set_vf_rate        = bnxt_set_vf_bw,
7761         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
7762         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
7763 #endif
7764 #ifdef CONFIG_NET_POLL_CONTROLLER
7765         .ndo_poll_controller    = bnxt_poll_controller,
7766 #endif
7767         .ndo_setup_tc           = bnxt_setup_tc,
7768 #ifdef CONFIG_RFS_ACCEL
7769         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
7770 #endif
7771         .ndo_udp_tunnel_add     = bnxt_udp_tunnel_add,
7772         .ndo_udp_tunnel_del     = bnxt_udp_tunnel_del,
7773         .ndo_bpf                = bnxt_xdp,
7774         .ndo_bridge_getlink     = bnxt_bridge_getlink,
7775         .ndo_bridge_setlink     = bnxt_bridge_setlink,
7776         .ndo_get_phys_port_name = bnxt_get_phys_port_name
7777 };
7778
7779 static void bnxt_remove_one(struct pci_dev *pdev)
7780 {
7781         struct net_device *dev = pci_get_drvdata(pdev);
7782         struct bnxt *bp = netdev_priv(dev);
7783
7784         if (BNXT_PF(bp)) {
7785                 bnxt_sriov_disable(bp);
7786                 bnxt_dl_unregister(bp);
7787         }
7788
7789         pci_disable_pcie_error_reporting(pdev);
7790         unregister_netdev(dev);
7791         bnxt_shutdown_tc(bp);
7792         bnxt_cancel_sp_work(bp);
7793         bp->sp_event = 0;
7794
7795         bnxt_clear_int_mode(bp);
7796         bnxt_hwrm_func_drv_unrgtr(bp);
7797         bnxt_free_hwrm_resources(bp);
7798         bnxt_free_hwrm_short_cmd_req(bp);
7799         bnxt_ethtool_free(bp);
7800         bnxt_dcb_free(bp);
7801         kfree(bp->edev);
7802         bp->edev = NULL;
7803         if (bp->xdp_prog)
7804                 bpf_prog_put(bp->xdp_prog);
7805         bnxt_cleanup_pci(bp);
7806         free_netdev(dev);
7807 }
7808
7809 static int bnxt_probe_phy(struct bnxt *bp)
7810 {
7811         int rc = 0;
7812         struct bnxt_link_info *link_info = &bp->link_info;
7813
7814         rc = bnxt_hwrm_phy_qcaps(bp);
7815         if (rc) {
7816                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
7817                            rc);
7818                 return rc;
7819         }
7820         mutex_init(&bp->link_lock);
7821
7822         rc = bnxt_update_link(bp, false);
7823         if (rc) {
7824                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
7825                            rc);
7826                 return rc;
7827         }
7828
7829         /* Older firmware does not have supported_auto_speeds, so assume
7830          * that all supported speeds can be autonegotiated.
7831          */
7832         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
7833                 link_info->support_auto_speeds = link_info->support_speeds;
7834
7835         /*initialize the ethool setting copy with NVM settings */
7836         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
7837                 link_info->autoneg = BNXT_AUTONEG_SPEED;
7838                 if (bp->hwrm_spec_code >= 0x10201) {
7839                         if (link_info->auto_pause_setting &
7840                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
7841                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
7842                 } else {
7843                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
7844                 }
7845                 link_info->advertising = link_info->auto_link_speeds;
7846         } else {
7847                 link_info->req_link_speed = link_info->force_link_speed;
7848                 link_info->req_duplex = link_info->duplex_setting;
7849         }
7850         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
7851                 link_info->req_flow_ctrl =
7852                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
7853         else
7854                 link_info->req_flow_ctrl = link_info->force_pause_setting;
7855         return rc;
7856 }
7857
7858 static int bnxt_get_max_irq(struct pci_dev *pdev)
7859 {
7860         u16 ctrl;
7861
7862         if (!pdev->msix_cap)
7863                 return 1;
7864
7865         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
7866         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
7867 }
7868
7869 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
7870                                 int *max_cp)
7871 {
7872         int max_ring_grps = 0;
7873
7874 #ifdef CONFIG_BNXT_SRIOV
7875         if (!BNXT_PF(bp)) {
7876                 *max_tx = bp->vf.max_tx_rings;
7877                 *max_rx = bp->vf.max_rx_rings;
7878                 *max_cp = min_t(int, bp->vf.max_irqs, bp->vf.max_cp_rings);
7879                 *max_cp = min_t(int, *max_cp, bp->vf.max_stat_ctxs);
7880                 max_ring_grps = bp->vf.max_hw_ring_grps;
7881         } else
7882 #endif
7883         {
7884                 *max_tx = bp->pf.max_tx_rings;
7885                 *max_rx = bp->pf.max_rx_rings;
7886                 *max_cp = min_t(int, bp->pf.max_irqs, bp->pf.max_cp_rings);
7887                 *max_cp = min_t(int, *max_cp, bp->pf.max_stat_ctxs);
7888                 max_ring_grps = bp->pf.max_hw_ring_grps;
7889         }
7890         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
7891                 *max_cp -= 1;
7892                 *max_rx -= 2;
7893         }
7894         if (bp->flags & BNXT_FLAG_AGG_RINGS)
7895                 *max_rx >>= 1;
7896         *max_rx = min_t(int, *max_rx, max_ring_grps);
7897 }
7898
7899 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
7900 {
7901         int rx, tx, cp;
7902
7903         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
7904         if (!rx || !tx || !cp)
7905                 return -ENOMEM;
7906
7907         *max_rx = rx;
7908         *max_tx = tx;
7909         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
7910 }
7911
7912 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
7913                                bool shared)
7914 {
7915         int rc;
7916
7917         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
7918         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
7919                 /* Not enough rings, try disabling agg rings. */
7920                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
7921                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
7922                 if (rc)
7923                         return rc;
7924                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
7925                 bp->dev->hw_features &= ~NETIF_F_LRO;
7926                 bp->dev->features &= ~NETIF_F_LRO;
7927                 bnxt_set_ring_params(bp);
7928         }
7929
7930         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
7931                 int max_cp, max_stat, max_irq;
7932
7933                 /* Reserve minimum resources for RoCE */
7934                 max_cp = bnxt_get_max_func_cp_rings(bp);
7935                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
7936                 max_irq = bnxt_get_max_func_irqs(bp);
7937                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
7938                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
7939                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
7940                         return 0;
7941
7942                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
7943                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
7944                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
7945                 max_cp = min_t(int, max_cp, max_irq);
7946                 max_cp = min_t(int, max_cp, max_stat);
7947                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
7948                 if (rc)
7949                         rc = 0;
7950         }
7951         return rc;
7952 }
7953
7954 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
7955 {
7956         int dflt_rings, max_rx_rings, max_tx_rings, rc;
7957
7958         if (sh)
7959                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
7960         dflt_rings = netif_get_num_default_rss_queues();
7961         /* Reduce default rings to reduce memory usage on multi-port cards */
7962         if (bp->port_count > 1)
7963                 dflt_rings = min_t(int, dflt_rings, 4);
7964         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
7965         if (rc)
7966                 return rc;
7967         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
7968         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
7969
7970         rc = bnxt_hwrm_reserve_tx_rings(bp, &bp->tx_nr_rings_per_tc);
7971         if (rc)
7972                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
7973
7974         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
7975         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
7976                                bp->tx_nr_rings + bp->rx_nr_rings;
7977         bp->num_stat_ctxs = bp->cp_nr_rings;
7978         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7979                 bp->rx_nr_rings++;
7980                 bp->cp_nr_rings++;
7981         }
7982         return rc;
7983 }
7984
7985 void bnxt_restore_pf_fw_resources(struct bnxt *bp)
7986 {
7987         ASSERT_RTNL();
7988         bnxt_hwrm_func_qcaps(bp);
7989         bnxt_subtract_ulp_resources(bp, BNXT_ROCE_ULP);
7990 }
7991
7992 static int bnxt_init_mac_addr(struct bnxt *bp)
7993 {
7994         int rc = 0;
7995
7996         if (BNXT_PF(bp)) {
7997                 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
7998         } else {
7999 #ifdef CONFIG_BNXT_SRIOV
8000                 struct bnxt_vf_info *vf = &bp->vf;
8001
8002                 if (is_valid_ether_addr(vf->mac_addr)) {
8003                         /* overwrite netdev dev_adr with admin VF MAC */
8004                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
8005                 } else {
8006                         eth_hw_addr_random(bp->dev);
8007                         rc = bnxt_approve_mac(bp, bp->dev->dev_addr);
8008                 }
8009 #endif
8010         }
8011         return rc;
8012 }
8013
8014 static void bnxt_parse_log_pcie_link(struct bnxt *bp)
8015 {
8016         enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN;
8017         enum pci_bus_speed speed = PCI_SPEED_UNKNOWN;
8018
8019         if (pcie_get_minimum_link(pci_physfn(bp->pdev), &speed, &width) ||
8020             speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
8021                 netdev_info(bp->dev, "Failed to determine PCIe Link Info\n");
8022         else
8023                 netdev_info(bp->dev, "PCIe: Speed %s Width x%d\n",
8024                             speed == PCIE_SPEED_2_5GT ? "2.5GT/s" :
8025                             speed == PCIE_SPEED_5_0GT ? "5.0GT/s" :
8026                             speed == PCIE_SPEED_8_0GT ? "8.0GT/s" :
8027                             "Unknown", width);
8028 }
8029
8030 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
8031 {
8032         static int version_printed;
8033         struct net_device *dev;
8034         struct bnxt *bp;
8035         int rc, max_irqs;
8036
8037         if (pci_is_bridge(pdev))
8038                 return -ENODEV;
8039
8040         if (version_printed++ == 0)
8041                 pr_info("%s", version);
8042
8043         max_irqs = bnxt_get_max_irq(pdev);
8044         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
8045         if (!dev)
8046                 return -ENOMEM;
8047
8048         bp = netdev_priv(dev);
8049
8050         if (bnxt_vf_pciid(ent->driver_data))
8051                 bp->flags |= BNXT_FLAG_VF;
8052
8053         if (pdev->msix_cap)
8054                 bp->flags |= BNXT_FLAG_MSIX_CAP;
8055
8056         rc = bnxt_init_board(pdev, dev);
8057         if (rc < 0)
8058                 goto init_err_free;
8059
8060         dev->netdev_ops = &bnxt_netdev_ops;
8061         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
8062         dev->ethtool_ops = &bnxt_ethtool_ops;
8063         SWITCHDEV_SET_OPS(dev, &bnxt_switchdev_ops);
8064         pci_set_drvdata(pdev, dev);
8065
8066         rc = bnxt_alloc_hwrm_resources(bp);
8067         if (rc)
8068                 goto init_err_pci_clean;
8069
8070         mutex_init(&bp->hwrm_cmd_lock);
8071         rc = bnxt_hwrm_ver_get(bp);
8072         if (rc)
8073                 goto init_err_pci_clean;
8074
8075         if (bp->flags & BNXT_FLAG_SHORT_CMD) {
8076                 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
8077                 if (rc)
8078                         goto init_err_pci_clean;
8079         }
8080
8081         rc = bnxt_hwrm_func_reset(bp);
8082         if (rc)
8083                 goto init_err_pci_clean;
8084
8085         bnxt_hwrm_fw_set_time(bp);
8086
8087         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
8088                            NETIF_F_TSO | NETIF_F_TSO6 |
8089                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
8090                            NETIF_F_GSO_IPXIP4 |
8091                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
8092                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
8093                            NETIF_F_RXCSUM | NETIF_F_GRO;
8094
8095         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
8096                 dev->hw_features |= NETIF_F_LRO;
8097
8098         dev->hw_enc_features =
8099                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
8100                         NETIF_F_TSO | NETIF_F_TSO6 |
8101                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
8102                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
8103                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
8104         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
8105                                     NETIF_F_GSO_GRE_CSUM;
8106         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
8107         dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
8108                             NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
8109         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
8110         dev->priv_flags |= IFF_UNICAST_FLT;
8111
8112 #ifdef CONFIG_BNXT_SRIOV
8113         init_waitqueue_head(&bp->sriov_cfg_wait);
8114         mutex_init(&bp->sriov_lock);
8115 #endif
8116         bp->gro_func = bnxt_gro_func_5730x;
8117         if (BNXT_CHIP_P4_PLUS(bp))
8118                 bp->gro_func = bnxt_gro_func_5731x;
8119         else
8120                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
8121
8122         rc = bnxt_hwrm_func_drv_rgtr(bp);
8123         if (rc)
8124                 goto init_err_pci_clean;
8125
8126         rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0);
8127         if (rc)
8128                 goto init_err_pci_clean;
8129
8130         bp->ulp_probe = bnxt_ulp_probe;
8131
8132         /* Get the MAX capabilities for this function */
8133         rc = bnxt_hwrm_func_qcaps(bp);
8134         if (rc) {
8135                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
8136                            rc);
8137                 rc = -1;
8138                 goto init_err_pci_clean;
8139         }
8140         rc = bnxt_init_mac_addr(bp);
8141         if (rc) {
8142                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
8143                 rc = -EADDRNOTAVAIL;
8144                 goto init_err_pci_clean;
8145         }
8146         rc = bnxt_hwrm_queue_qportcfg(bp);
8147         if (rc) {
8148                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n",
8149                            rc);
8150                 rc = -1;
8151                 goto init_err_pci_clean;
8152         }
8153
8154         bnxt_hwrm_func_qcfg(bp);
8155         bnxt_hwrm_port_led_qcaps(bp);
8156         bnxt_ethtool_init(bp);
8157         bnxt_dcb_init(bp);
8158
8159         /* MTU range: 60 - FW defined max */
8160         dev->min_mtu = ETH_ZLEN;
8161         dev->max_mtu = bp->max_mtu;
8162
8163         rc = bnxt_probe_phy(bp);
8164         if (rc)
8165                 goto init_err_pci_clean;
8166
8167         bnxt_set_rx_skb_mode(bp, false);
8168         bnxt_set_tpa_flags(bp);
8169         bnxt_set_ring_params(bp);
8170         bnxt_set_max_func_irqs(bp, max_irqs);
8171         rc = bnxt_set_dflt_rings(bp, true);
8172         if (rc) {
8173                 netdev_err(bp->dev, "Not enough rings available.\n");
8174                 rc = -ENOMEM;
8175                 goto init_err_pci_clean;
8176         }
8177
8178         /* Default RSS hash cfg. */
8179         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
8180                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
8181                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
8182                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
8183         if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
8184                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
8185                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
8186                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
8187         }
8188
8189         bnxt_hwrm_vnic_qcaps(bp);
8190         if (bnxt_rfs_supported(bp)) {
8191                 dev->hw_features |= NETIF_F_NTUPLE;
8192                 if (bnxt_rfs_capable(bp)) {
8193                         bp->flags |= BNXT_FLAG_RFS;
8194                         dev->features |= NETIF_F_NTUPLE;
8195                 }
8196         }
8197
8198         if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
8199                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
8200
8201         rc = bnxt_init_int_mode(bp);
8202         if (rc)
8203                 goto init_err_pci_clean;
8204
8205         bnxt_get_wol_settings(bp);
8206         if (bp->flags & BNXT_FLAG_WOL_CAP)
8207                 device_set_wakeup_enable(&pdev->dev, bp->wol);
8208         else
8209                 device_set_wakeup_capable(&pdev->dev, false);
8210
8211         if (BNXT_PF(bp)) {
8212                 if (!bnxt_pf_wq) {
8213                         bnxt_pf_wq =
8214                                 create_singlethread_workqueue("bnxt_pf_wq");
8215                         if (!bnxt_pf_wq) {
8216                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
8217                                 goto init_err_pci_clean;
8218                         }
8219                 }
8220                 bnxt_init_tc(bp);
8221         }
8222
8223         rc = register_netdev(dev);
8224         if (rc)
8225                 goto init_err_cleanup_tc;
8226
8227         if (BNXT_PF(bp))
8228                 bnxt_dl_register(bp);
8229
8230         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
8231                     board_info[ent->driver_data].name,
8232                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
8233
8234         bnxt_parse_log_pcie_link(bp);
8235
8236         return 0;
8237
8238 init_err_cleanup_tc:
8239         bnxt_shutdown_tc(bp);
8240         bnxt_clear_int_mode(bp);
8241
8242 init_err_pci_clean:
8243         bnxt_cleanup_pci(bp);
8244
8245 init_err_free:
8246         free_netdev(dev);
8247         return rc;
8248 }
8249
8250 static void bnxt_shutdown(struct pci_dev *pdev)
8251 {
8252         struct net_device *dev = pci_get_drvdata(pdev);
8253         struct bnxt *bp;
8254
8255         if (!dev)
8256                 return;
8257
8258         rtnl_lock();
8259         bp = netdev_priv(dev);
8260         if (!bp)
8261                 goto shutdown_exit;
8262
8263         if (netif_running(dev))
8264                 dev_close(dev);
8265
8266         bnxt_ulp_shutdown(bp);
8267
8268         if (system_state == SYSTEM_POWER_OFF) {
8269                 bnxt_clear_int_mode(bp);
8270                 pci_wake_from_d3(pdev, bp->wol);
8271                 pci_set_power_state(pdev, PCI_D3hot);
8272         }
8273
8274 shutdown_exit:
8275         rtnl_unlock();
8276 }
8277
8278 #ifdef CONFIG_PM_SLEEP
8279 static int bnxt_suspend(struct device *device)
8280 {
8281         struct pci_dev *pdev = to_pci_dev(device);
8282         struct net_device *dev = pci_get_drvdata(pdev);
8283         struct bnxt *bp = netdev_priv(dev);
8284         int rc = 0;
8285
8286         rtnl_lock();
8287         if (netif_running(dev)) {
8288                 netif_device_detach(dev);
8289                 rc = bnxt_close(dev);
8290         }
8291         bnxt_hwrm_func_drv_unrgtr(bp);
8292         rtnl_unlock();
8293         return rc;
8294 }
8295
8296 static int bnxt_resume(struct device *device)
8297 {
8298         struct pci_dev *pdev = to_pci_dev(device);
8299         struct net_device *dev = pci_get_drvdata(pdev);
8300         struct bnxt *bp = netdev_priv(dev);
8301         int rc = 0;
8302
8303         rtnl_lock();
8304         if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) {
8305                 rc = -ENODEV;
8306                 goto resume_exit;
8307         }
8308         rc = bnxt_hwrm_func_reset(bp);
8309         if (rc) {
8310                 rc = -EBUSY;
8311                 goto resume_exit;
8312         }
8313         bnxt_get_wol_settings(bp);
8314         if (netif_running(dev)) {
8315                 rc = bnxt_open(dev);
8316                 if (!rc)
8317                         netif_device_attach(dev);
8318         }
8319
8320 resume_exit:
8321         rtnl_unlock();
8322         return rc;
8323 }
8324
8325 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
8326 #define BNXT_PM_OPS (&bnxt_pm_ops)
8327
8328 #else
8329
8330 #define BNXT_PM_OPS NULL
8331
8332 #endif /* CONFIG_PM_SLEEP */
8333
8334 /**
8335  * bnxt_io_error_detected - called when PCI error is detected
8336  * @pdev: Pointer to PCI device
8337  * @state: The current pci connection state
8338  *
8339  * This function is called after a PCI bus error affecting
8340  * this device has been detected.
8341  */
8342 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
8343                                                pci_channel_state_t state)
8344 {
8345         struct net_device *netdev = pci_get_drvdata(pdev);
8346         struct bnxt *bp = netdev_priv(netdev);
8347
8348         netdev_info(netdev, "PCI I/O error detected\n");
8349
8350         rtnl_lock();
8351         netif_device_detach(netdev);
8352
8353         bnxt_ulp_stop(bp);
8354
8355         if (state == pci_channel_io_perm_failure) {
8356                 rtnl_unlock();
8357                 return PCI_ERS_RESULT_DISCONNECT;
8358         }
8359
8360         if (netif_running(netdev))
8361                 bnxt_close(netdev);
8362
8363         pci_disable_device(pdev);
8364         rtnl_unlock();
8365
8366         /* Request a slot slot reset. */
8367         return PCI_ERS_RESULT_NEED_RESET;
8368 }
8369
8370 /**
8371  * bnxt_io_slot_reset - called after the pci bus has been reset.
8372  * @pdev: Pointer to PCI device
8373  *
8374  * Restart the card from scratch, as if from a cold-boot.
8375  * At this point, the card has exprienced a hard reset,
8376  * followed by fixups by BIOS, and has its config space
8377  * set up identically to what it was at cold boot.
8378  */
8379 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
8380 {
8381         struct net_device *netdev = pci_get_drvdata(pdev);
8382         struct bnxt *bp = netdev_priv(netdev);
8383         int err = 0;
8384         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
8385
8386         netdev_info(bp->dev, "PCI Slot Reset\n");
8387
8388         rtnl_lock();
8389
8390         if (pci_enable_device(pdev)) {
8391                 dev_err(&pdev->dev,
8392                         "Cannot re-enable PCI device after reset.\n");
8393         } else {
8394                 pci_set_master(pdev);
8395
8396                 err = bnxt_hwrm_func_reset(bp);
8397                 if (!err && netif_running(netdev))
8398                         err = bnxt_open(netdev);
8399
8400                 if (!err) {
8401                         result = PCI_ERS_RESULT_RECOVERED;
8402                         bnxt_ulp_start(bp);
8403                 }
8404         }
8405
8406         if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
8407                 dev_close(netdev);
8408
8409         rtnl_unlock();
8410
8411         err = pci_cleanup_aer_uncorrect_error_status(pdev);
8412         if (err) {
8413                 dev_err(&pdev->dev,
8414                         "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
8415                          err); /* non-fatal, continue */
8416         }
8417
8418         return PCI_ERS_RESULT_RECOVERED;
8419 }
8420
8421 /**
8422  * bnxt_io_resume - called when traffic can start flowing again.
8423  * @pdev: Pointer to PCI device
8424  *
8425  * This callback is called when the error recovery driver tells
8426  * us that its OK to resume normal operation.
8427  */
8428 static void bnxt_io_resume(struct pci_dev *pdev)
8429 {
8430         struct net_device *netdev = pci_get_drvdata(pdev);
8431
8432         rtnl_lock();
8433
8434         netif_device_attach(netdev);
8435
8436         rtnl_unlock();
8437 }
8438
8439 static const struct pci_error_handlers bnxt_err_handler = {
8440         .error_detected = bnxt_io_error_detected,
8441         .slot_reset     = bnxt_io_slot_reset,
8442         .resume         = bnxt_io_resume
8443 };
8444
8445 static struct pci_driver bnxt_pci_driver = {
8446         .name           = DRV_MODULE_NAME,
8447         .id_table       = bnxt_pci_tbl,
8448         .probe          = bnxt_init_one,
8449         .remove         = bnxt_remove_one,
8450         .shutdown       = bnxt_shutdown,
8451         .driver.pm      = BNXT_PM_OPS,
8452         .err_handler    = &bnxt_err_handler,
8453 #if defined(CONFIG_BNXT_SRIOV)
8454         .sriov_configure = bnxt_sriov_configure,
8455 #endif
8456 };
8457
8458 static int __init bnxt_init(void)
8459 {
8460         return pci_register_driver(&bnxt_pci_driver);
8461 }
8462
8463 static void __exit bnxt_exit(void)
8464 {
8465         pci_unregister_driver(&bnxt_pci_driver);
8466         if (bnxt_pf_wq)
8467                 destroy_workqueue(bnxt_pf_wq);
8468 }
8469
8470 module_init(bnxt_init);
8471 module_exit(bnxt_exit);