drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2017 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/if.h>
  35 #include <linux/if_vlan.h>
  36 #include <linux/if_bridge.h>
  37 #include <linux/rtc.h>
  38 #include <linux/bpf.h>
  39 #include <net/ip.h>
  40 #include <net/tcp.h>
  41 #include <net/udp.h>
  42 #include <net/checksum.h>
  43 #include <net/ip6_checksum.h>
  44 #include <net/udp_tunnel.h>
  45 #include <linux/workqueue.h>
  46 #include <linux/prefetch.h>
  47 #include <linux/cache.h>
  48 #include <linux/log2.h>
  49 #include <linux/aer.h>
  50 #include <linux/bitmap.h>
  51 #include <linux/cpu_rmap.h>
  52 #include <linux/cpumask.h>
  53 #include <net/pkt_cls.h>
  54
  55 #include "bnxt_hsi.h"
  56 #include "bnxt.h"
  57 #include "bnxt_ulp.h"
  58 #include "bnxt_sriov.h"
  59 #include "bnxt_ethtool.h"
  60 #include "bnxt_dcb.h"
  61 #include "bnxt_xdp.h"
  62 #include "bnxt_vfr.h"
  63 #include "bnxt_tc.h"
  64
  65 #define BNXT_TX_TIMEOUT         (5 * HZ)
  66
  67 static const char version[] =
  68         "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
  69
  70 MODULE_LICENSE("GPL");
  71 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  72 MODULE_VERSION(DRV_MODULE_VERSION);
  73
  74 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  75 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  76 #define BNXT_RX_COPY_THRESH 256
  77
  78 #define BNXT_TX_PUSH_THRESH 164
  79
  80 enum board_idx {
  81         BCM57301,
  82         BCM57302,
  83         BCM57304,
  84         BCM57417_NPAR,
  85         BCM58700,
  86         BCM57311,
  87         BCM57312,
  88         BCM57402,
  89         BCM57404,
  90         BCM57406,
  91         BCM57402_NPAR,
  92         BCM57407,
  93         BCM57412,
  94         BCM57414,
  95         BCM57416,
  96         BCM57417,
  97         BCM57412_NPAR,
  98         BCM57314,
  99         BCM57417_SFP,
 100         BCM57416_SFP,
 101         BCM57404_NPAR,
 102         BCM57406_NPAR,
 103         BCM57407_SFP,
 104         BCM57407_NPAR,
 105         BCM57414_NPAR,
 106         BCM57416_NPAR,
 107         BCM57452,
 108         BCM57454,
 109         BCM58802,
 110         BCM58808,
 111         NETXTREME_E_VF,
 112         NETXTREME_C_VF,
 113 };
 114
 115 /* indexed by enum above */
 116 static const struct {
 117         char *name;
 118 } board_info[] = {
 119         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
 120         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
 121         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 122         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
 123         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
 124         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
 125         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
 126         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 127         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 128         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 129         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 130         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 131         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 132         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 133         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 134         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 135         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 136         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 137         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 138         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 139         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 140         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 141         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 142         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 143         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 144         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 145         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 146         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 147         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 148         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 149         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 150         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 151 };
 152
 153 static const struct pci_device_id bnxt_pci_tbl[] = {
 154         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 155         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 156         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 157         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 158         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 159         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 160         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 161         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 162         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 163         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 164         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 165         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 166         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 167         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 168         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 169         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 170         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 171         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 172         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 173         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 174         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 175         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 176         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 177         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 178         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 179         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 180         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 181         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 182         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 183         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 184         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 185         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 186         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 187         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 188 #ifdef CONFIG_BNXT_SRIOV
 189         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 190         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 191         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 192         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 193         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 194         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 195         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 196         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 197 #endif
 198         { 0 }
 199 };
 200
 201 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 202
 203 static const u16 bnxt_vf_req_snif[] = {
 204         HWRM_FUNC_CFG,
 205         HWRM_PORT_PHY_QCFG,
 206         HWRM_CFA_L2_FILTER_ALLOC,
 207 };
 208
 209 static const u16 bnxt_async_events_arr[] = {
 210         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 211         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 212         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 213         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 214         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 215 };
 216
 217 static struct workqueue_struct *bnxt_pf_wq;
 218
 219 static bool bnxt_vf_pciid(enum board_idx idx)
 220 {
 221         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF);
 222 }
 223
 224 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 225 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 226 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 227
 228 #define BNXT_CP_DB_REARM(db, raw_cons)                                  \
 229                 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db)
 230
 231 #define BNXT_CP_DB(db, raw_cons)                                        \
 232                 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db)
 233
 234 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 235                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 236
 237 const u16 bnxt_lhint_arr[] = {
 238         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 239         TX_BD_FLAGS_LHINT_512_TO_1023,
 240         TX_BD_FLAGS_LHINT_1024_TO_2047,
 241         TX_BD_FLAGS_LHINT_1024_TO_2047,
 242         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 243         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 244         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 245         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 246         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 247         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 248         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 249         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 250         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 251         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 252         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 253         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 254         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 255         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 256         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 257 };
 258
 259 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 260 {
 261         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 262
 263         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 264                 return 0;
 265
 266         return md_dst->u.port_info.port_id;
 267 }
 268
 269 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 270 {
 271         struct bnxt *bp = netdev_priv(dev);
 272         struct tx_bd *txbd;
 273         struct tx_bd_ext *txbd1;
 274         struct netdev_queue *txq;
 275         int i;
 276         dma_addr_t mapping;
 277         unsigned int length, pad = 0;
 278         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 279         u16 prod, last_frag;
 280         struct pci_dev *pdev = bp->pdev;
 281         struct bnxt_tx_ring_info *txr;
 282         struct bnxt_sw_tx_bd *tx_buf;
 283
 284         i = skb_get_queue_mapping(skb);
 285         if (unlikely(i >= bp->tx_nr_rings)) {
 286                 dev_kfree_skb_any(skb);
 287                 return NETDEV_TX_OK;
 288         }
 289
 290         txq = netdev_get_tx_queue(dev, i);
 291         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 292         prod = txr->tx_prod;
 293
 294         free_size = bnxt_tx_avail(bp, txr);
 295         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 296                 netif_tx_stop_queue(txq);
 297                 return NETDEV_TX_BUSY;
 298         }
 299
 300         length = skb->len;
 301         len = skb_headlen(skb);
 302         last_frag = skb_shinfo(skb)->nr_frags;
 303
 304         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 305
 306         txbd->tx_bd_opaque = prod;
 307
 308         tx_buf = &txr->tx_buf_ring[prod];
 309         tx_buf->skb = skb;
 310         tx_buf->nr_frags = last_frag;
 311
 312         vlan_tag_flags = 0;
 313         cfa_action = bnxt_xmit_get_cfa_action(skb);
 314         if (skb_vlan_tag_present(skb)) {
 315                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 316                                  skb_vlan_tag_get(skb);
 317                 /* Currently supports 8021Q, 8021AD vlan offloads
 318                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 319                  */
 320                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 321                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 322         }
 323
 324         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 325                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 326                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 327                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 328                 void *pdata = tx_push_buf->data;
 329                 u64 *end;
 330                 int j, push_len;
 331
 332                 /* Set COAL_NOW to be ready quickly for the next push */
 333                 tx_push->tx_bd_len_flags_type =
 334                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 335                                         TX_BD_TYPE_LONG_TX_BD |
 336                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 337                                         TX_BD_FLAGS_COAL_NOW |
 338                                         TX_BD_FLAGS_PACKET_END |
 339                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 340
 341                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 342                         tx_push1->tx_bd_hsize_lflags =
 343                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 344                 else
 345                         tx_push1->tx_bd_hsize_lflags = 0;
 346
 347                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 348                 tx_push1->tx_bd_cfa_action =
 349                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 350
 351                 end = pdata + length;
 352                 end = PTR_ALIGN(end, 8) - 1;
 353                 *end = 0;
 354
 355                 skb_copy_from_linear_data(skb, pdata, len);
 356                 pdata += len;
 357                 for (j = 0; j < last_frag; j++) {
 358                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 359                         void *fptr;
 360
 361                         fptr = skb_frag_address_safe(frag);
 362                         if (!fptr)
 363                                 goto normal_tx;
 364
 365                         memcpy(pdata, fptr, skb_frag_size(frag));
 366                         pdata += skb_frag_size(frag);
 367                 }
 368
 369                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 370                 txbd->tx_bd_haddr = txr->data_mapping;
 371                 prod = NEXT_TX(prod);
 372                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 373                 memcpy(txbd, tx_push1, sizeof(*txbd));
 374                 prod = NEXT_TX(prod);
 375                 tx_push->doorbell =
 376                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 377                 txr->tx_prod = prod;
 378
 379                 tx_buf->is_push = 1;
 380                 netdev_tx_sent_queue(txq, skb->len);
 381                 wmb();  /* Sync is_push and byte queue before pushing data */
 382
 383                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 384                 if (push_len > 16) {
 385                         __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16);
 386                         __iowrite32_copy(txr->tx_doorbell + 4, tx_push_buf + 1,
 387                                          (push_len - 16) << 1);
 388                 } else {
 389                         __iowrite64_copy(txr->tx_doorbell, tx_push_buf,
 390                                          push_len);
 391                 }
 392
 393                 goto tx_done;
 394         }
 395
 396 normal_tx:
 397         if (length < BNXT_MIN_PKT_SIZE) {
 398                 pad = BNXT_MIN_PKT_SIZE - length;
 399                 if (skb_pad(skb, pad)) {
 400                         /* SKB already freed. */
 401                         tx_buf->skb = NULL;
 402                         return NETDEV_TX_OK;
 403                 }
 404                 length = BNXT_MIN_PKT_SIZE;
 405         }
 406
 407         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 408
 409         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 410                 dev_kfree_skb_any(skb);
 411                 tx_buf->skb = NULL;
 412                 return NETDEV_TX_OK;
 413         }
 414
 415         dma_unmap_addr_set(tx_buf, mapping, mapping);
 416         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 417                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 418
 419         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 420
 421         prod = NEXT_TX(prod);
 422         txbd1 = (struct tx_bd_ext *)
 423                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 424
 425         txbd1->tx_bd_hsize_lflags = 0;
 426         if (skb_is_gso(skb)) {
 427                 u32 hdr_len;
 428
 429                 if (skb->encapsulation)
 430                         hdr_len = skb_inner_network_offset(skb) +
 431                                 skb_inner_network_header_len(skb) +
 432                                 inner_tcp_hdrlen(skb);
 433                 else
 434                         hdr_len = skb_transport_offset(skb) +
 435                                 tcp_hdrlen(skb);
 436
 437                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 438                                         TX_BD_FLAGS_T_IPID |
 439                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 440                 length = skb_shinfo(skb)->gso_size;
 441                 txbd1->tx_bd_mss = cpu_to_le32(length);
 442                 length += hdr_len;
 443         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 444                 txbd1->tx_bd_hsize_lflags =
 445                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 446                 txbd1->tx_bd_mss = 0;
 447         }
 448
 449         length >>= 9;
 450         flags |= bnxt_lhint_arr[length];
 451         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 452
 453         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 454         txbd1->tx_bd_cfa_action =
 455                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 456         for (i = 0; i < last_frag; i++) {
 457                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 458
 459                 prod = NEXT_TX(prod);
 460                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 461
 462                 len = skb_frag_size(frag);
 463                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 464                                            DMA_TO_DEVICE);
 465
 466                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 467                         goto tx_dma_error;
 468
 469                 tx_buf = &txr->tx_buf_ring[prod];
 470                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 471
 472                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 473
 474                 flags = len << TX_BD_LEN_SHIFT;
 475                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 476         }
 477
 478         flags &= ~TX_BD_LEN;
 479         txbd->tx_bd_len_flags_type =
 480                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 481                             TX_BD_FLAGS_PACKET_END);
 482
 483         netdev_tx_sent_queue(txq, skb->len);
 484
 485         /* Sync BD data before updating doorbell */
 486         wmb();
 487
 488         prod = NEXT_TX(prod);
 489         txr->tx_prod = prod;
 490
 491         if (!skb->xmit_more || netif_xmit_stopped(txq))
 492                 bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod);
 493
 494 tx_done:
 495
 496         mmiowb();
 497
 498         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 499                 if (skb->xmit_more && !tx_buf->is_push)
 500                         bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod);
 501
 502                 netif_tx_stop_queue(txq);
 503
 504                 /* netif_tx_stop_queue() must be done before checking
 505                  * tx index in bnxt_tx_avail() below, because in
 506                  * bnxt_tx_int(), we update tx index before checking for
 507                  * netif_tx_queue_stopped().
 508                  */
 509                 smp_mb();
 510                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 511                         netif_tx_wake_queue(txq);
 512         }
 513         return NETDEV_TX_OK;
 514
 515 tx_dma_error:
 516         last_frag = i;
 517
 518         /* start back at beginning and unmap skb */
 519         prod = txr->tx_prod;
 520         tx_buf = &txr->tx_buf_ring[prod];
 521         tx_buf->skb = NULL;
 522         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 523                          skb_headlen(skb), PCI_DMA_TODEVICE);
 524         prod = NEXT_TX(prod);
 525
 526         /* unmap remaining mapped pages */
 527         for (i = 0; i < last_frag; i++) {
 528                 prod = NEXT_TX(prod);
 529                 tx_buf = &txr->tx_buf_ring[prod];
 530                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 531                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 532                                PCI_DMA_TODEVICE);
 533         }
 534
 535         dev_kfree_skb_any(skb);
 536         return NETDEV_TX_OK;
 537 }
 538
 539 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 540 {
 541         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 542         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 543         u16 cons = txr->tx_cons;
 544         struct pci_dev *pdev = bp->pdev;
 545         int i;
 546         unsigned int tx_bytes = 0;
 547
 548         for (i = 0; i < nr_pkts; i++) {
 549                 struct bnxt_sw_tx_bd *tx_buf;
 550                 struct sk_buff *skb;
 551                 int j, last;
 552
 553                 tx_buf = &txr->tx_buf_ring[cons];
 554                 cons = NEXT_TX(cons);
 555                 skb = tx_buf->skb;
 556                 tx_buf->skb = NULL;
 557
 558                 if (tx_buf->is_push) {
 559                         tx_buf->is_push = 0;
 560                         goto next_tx_int;
 561                 }
 562
 563                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 564                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 565                 last = tx_buf->nr_frags;
 566
 567                 for (j = 0; j < last; j++) {
 568                         cons = NEXT_TX(cons);
 569                         tx_buf = &txr->tx_buf_ring[cons];
 570                         dma_unmap_page(
 571                                 &pdev->dev,
 572                                 dma_unmap_addr(tx_buf, mapping),
 573                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 574                                 PCI_DMA_TODEVICE);
 575                 }
 576
 577 next_tx_int:
 578                 cons = NEXT_TX(cons);
 579
 580                 tx_bytes += skb->len;
 581                 dev_kfree_skb_any(skb);
 582         }
 583
 584         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 585         txr->tx_cons = cons;
 586
 587         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 588          * before checking for netif_tx_queue_stopped().  Without the
 589          * memory barrier, there is a small possibility that bnxt_start_xmit()
 590          * will miss it and cause the queue to be stopped forever.
 591          */
 592         smp_mb();
 593
 594         if (unlikely(netif_tx_queue_stopped(txq)) &&
 595             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 596                 __netif_tx_lock(txq, smp_processor_id());
 597                 if (netif_tx_queue_stopped(txq) &&
 598                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 599                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 600                         netif_tx_wake_queue(txq);
 601                 __netif_tx_unlock(txq);
 602         }
 603 }
 604
 605 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 606                                          gfp_t gfp)
 607 {
 608         struct device *dev = &bp->pdev->dev;
 609         struct page *page;
 610
 611         page = alloc_page(gfp);
 612         if (!page)
 613                 return NULL;
 614
 615         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 616                                       DMA_ATTR_WEAK_ORDERING);
 617         if (dma_mapping_error(dev, *mapping)) {
 618                 __free_page(page);
 619                 return NULL;
 620         }
 621         *mapping += bp->rx_dma_offset;
 622         return page;
 623 }
 624
 625 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 626                                        gfp_t gfp)
 627 {
 628         u8 *data;
 629         struct pci_dev *pdev = bp->pdev;
 630
 631         data = kmalloc(bp->rx_buf_size, gfp);
 632         if (!data)
 633                 return NULL;
 634
 635         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 636                                         bp->rx_buf_use_size, bp->rx_dir,
 637                                         DMA_ATTR_WEAK_ORDERING);
 638
 639         if (dma_mapping_error(&pdev->dev, *mapping)) {
 640                 kfree(data);
 641                 data = NULL;
 642         }
 643         return data;
 644 }
 645
 646 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 647                        u16 prod, gfp_t gfp)
 648 {
 649         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 650         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 651         dma_addr_t mapping;
 652
 653         if (BNXT_RX_PAGE_MODE(bp)) {
 654                 struct page *page = __bnxt_alloc_rx_page(bp, &mapping, gfp);
 655
 656                 if (!page)
 657                         return -ENOMEM;
 658
 659                 rx_buf->data = page;
 660                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 661         } else {
 662                 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 663
 664                 if (!data)
 665                         return -ENOMEM;
 666
 667                 rx_buf->data = data;
 668                 rx_buf->data_ptr = data + bp->rx_offset;
 669         }
 670         rx_buf->mapping = mapping;
 671
 672         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 673         return 0;
 674 }
 675
 676 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 677 {
 678         u16 prod = rxr->rx_prod;
 679         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 680         struct rx_bd *cons_bd, *prod_bd;
 681
 682         prod_rx_buf = &rxr->rx_buf_ring[prod];
 683         cons_rx_buf = &rxr->rx_buf_ring[cons];
 684
 685         prod_rx_buf->data = data;
 686         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 687
 688         prod_rx_buf->mapping = cons_rx_buf->mapping;
 689
 690         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 691         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 692
 693         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 694 }
 695
 696 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 697 {
 698         u16 next, max = rxr->rx_agg_bmap_size;
 699
 700         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 701         if (next >= max)
 702                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 703         return next;
 704 }
 705
 706 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 707                                      struct bnxt_rx_ring_info *rxr,
 708                                      u16 prod, gfp_t gfp)
 709 {
 710         struct rx_bd *rxbd =
 711                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 712         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 713         struct pci_dev *pdev = bp->pdev;
 714         struct page *page;
 715         dma_addr_t mapping;
 716         u16 sw_prod = rxr->rx_sw_agg_prod;
 717         unsigned int offset = 0;
 718
 719         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 720                 page = rxr->rx_page;
 721                 if (!page) {
 722                         page = alloc_page(gfp);
 723                         if (!page)
 724                                 return -ENOMEM;
 725                         rxr->rx_page = page;
 726                         rxr->rx_page_offset = 0;
 727                 }
 728                 offset = rxr->rx_page_offset;
 729                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 730                 if (rxr->rx_page_offset == PAGE_SIZE)
 731                         rxr->rx_page = NULL;
 732                 else
 733                         get_page(page);
 734         } else {
 735                 page = alloc_page(gfp);
 736                 if (!page)
 737                         return -ENOMEM;
 738         }
 739
 740         mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 741                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
 742                                      DMA_ATTR_WEAK_ORDERING);
 743         if (dma_mapping_error(&pdev->dev, mapping)) {
 744                 __free_page(page);
 745                 return -EIO;
 746         }
 747
 748         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 749                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 750
 751         __set_bit(sw_prod, rxr->rx_agg_bmap);
 752         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 753         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 754
 755         rx_agg_buf->page = page;
 756         rx_agg_buf->offset = offset;
 757         rx_agg_buf->mapping = mapping;
 758         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 759         rxbd->rx_bd_opaque = sw_prod;
 760         return 0;
 761 }
 762
 763 static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons,
 764                                    u32 agg_bufs)
 765 {
 766         struct bnxt *bp = bnapi->bp;
 767         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 768         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 769         u16 prod = rxr->rx_agg_prod;
 770         u16 sw_prod = rxr->rx_sw_agg_prod;
 771         u32 i;
 772
 773         for (i = 0; i < agg_bufs; i++) {
 774                 u16 cons;
 775                 struct rx_agg_cmp *agg;
 776                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 777                 struct rx_bd *prod_bd;
 778                 struct page *page;
 779
 780                 agg = (struct rx_agg_cmp *)
 781                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 782                 cons = agg->rx_agg_cmp_opaque;
 783                 __clear_bit(cons, rxr->rx_agg_bmap);
 784
 785                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 786                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 787
 788                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 789                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 790                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 791
 792                 /* It is possible for sw_prod to be equal to cons, so
 793                  * set cons_rx_buf->page to NULL first.
 794                  */
 795                 page = cons_rx_buf->page;
 796                 cons_rx_buf->page = NULL;
 797                 prod_rx_buf->page = page;
 798                 prod_rx_buf->offset = cons_rx_buf->offset;
 799
 800                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 801
 802                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 803
 804                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 805                 prod_bd->rx_bd_opaque = sw_prod;
 806
 807                 prod = NEXT_RX_AGG(prod);
 808                 sw_prod = NEXT_RX_AGG(sw_prod);
 809                 cp_cons = NEXT_CMP(cp_cons);
 810         }
 811         rxr->rx_agg_prod = prod;
 812         rxr->rx_sw_agg_prod = sw_prod;
 813 }
 814
 815 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 816                                         struct bnxt_rx_ring_info *rxr,
 817                                         u16 cons, void *data, u8 *data_ptr,
 818                                         dma_addr_t dma_addr,
 819                                         unsigned int offset_and_len)
 820 {
 821         unsigned int payload = offset_and_len >> 16;
 822         unsigned int len = offset_and_len & 0xffff;
 823         struct skb_frag_struct *frag;
 824         struct page *page = data;
 825         u16 prod = rxr->rx_prod;
 826         struct sk_buff *skb;
 827         int off, err;
 828
 829         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 830         if (unlikely(err)) {
 831                 bnxt_reuse_rx_data(rxr, cons, data);
 832                 return NULL;
 833         }
 834         dma_addr -= bp->rx_dma_offset;
 835         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 836                              DMA_ATTR_WEAK_ORDERING);
 837
 838         if (unlikely(!payload))
 839                 payload = eth_get_headlen(data_ptr, len);
 840
 841         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
 842         if (!skb) {
 843                 __free_page(page);
 844                 return NULL;
 845         }
 846
 847         off = (void *)data_ptr - page_address(page);
 848         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
 849         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
 850                payload + NET_IP_ALIGN);
 851
 852         frag = &skb_shinfo(skb)->frags[0];
 853         skb_frag_size_sub(frag, payload);
 854         frag->page_offset += payload;
 855         skb->data_len -= payload;
 856         skb->tail += payload;
 857
 858         return skb;
 859 }
 860
 861 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 862                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 863                                    void *data, u8 *data_ptr,
 864                                    dma_addr_t dma_addr,
 865                                    unsigned int offset_and_len)
 866 {
 867         u16 prod = rxr->rx_prod;
 868         struct sk_buff *skb;
 869         int err;
 870
 871         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 872         if (unlikely(err)) {
 873                 bnxt_reuse_rx_data(rxr, cons, data);
 874                 return NULL;
 875         }
 876
 877         skb = build_skb(data, 0);
 878         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 879                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
 880         if (!skb) {
 881                 kfree(data);
 882                 return NULL;
 883         }
 884
 885         skb_reserve(skb, bp->rx_offset);
 886         skb_put(skb, offset_and_len & 0xffff);
 887         return skb;
 888 }
 889
 890 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi,
 891                                      struct sk_buff *skb, u16 cp_cons,
 892                                      u32 agg_bufs)
 893 {
 894         struct pci_dev *pdev = bp->pdev;
 895         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 896         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 897         u16 prod = rxr->rx_agg_prod;
 898         u32 i;
 899
 900         for (i = 0; i < agg_bufs; i++) {
 901                 u16 cons, frag_len;
 902                 struct rx_agg_cmp *agg;
 903                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
 904                 struct page *page;
 905                 dma_addr_t mapping;
 906
 907                 agg = (struct rx_agg_cmp *)
 908                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 909                 cons = agg->rx_agg_cmp_opaque;
 910                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
 911                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
 912
 913                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 914                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
 915                                    cons_rx_buf->offset, frag_len);
 916                 __clear_bit(cons, rxr->rx_agg_bmap);
 917
 918                 /* It is possible for bnxt_alloc_rx_page() to allocate
 919                  * a sw_prod index that equals the cons index, so we
 920                  * need to clear the cons entry now.
 921                  */
 922                 mapping = cons_rx_buf->mapping;
 923                 page = cons_rx_buf->page;
 924                 cons_rx_buf->page = NULL;
 925
 926                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
 927                         struct skb_shared_info *shinfo;
 928                         unsigned int nr_frags;
 929
 930                         shinfo = skb_shinfo(skb);
 931                         nr_frags = --shinfo->nr_frags;
 932                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
 933
 934                         dev_kfree_skb(skb);
 935
 936                         cons_rx_buf->page = page;
 937
 938                         /* Update prod since possibly some pages have been
 939                          * allocated already.
 940                          */
 941                         rxr->rx_agg_prod = prod;
 942                         bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i);
 943                         return NULL;
 944                 }
 945
 946                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
 947                                      PCI_DMA_FROMDEVICE,
 948                                      DMA_ATTR_WEAK_ORDERING);
 949
 950                 skb->data_len += frag_len;
 951                 skb->len += frag_len;
 952                 skb->truesize += PAGE_SIZE;
 953
 954                 prod = NEXT_RX_AGG(prod);
 955                 cp_cons = NEXT_CMP(cp_cons);
 956         }
 957         rxr->rx_agg_prod = prod;
 958         return skb;
 959 }
 960
 961 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 962                                u8 agg_bufs, u32 *raw_cons)
 963 {
 964         u16 last;
 965         struct rx_agg_cmp *agg;
 966
 967         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
 968         last = RING_CMP(*raw_cons);
 969         agg = (struct rx_agg_cmp *)
 970                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
 971         return RX_AGG_CMP_VALID(agg, *raw_cons);
 972 }
 973
 974 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
 975                                             unsigned int len,
 976                                             dma_addr_t mapping)
 977 {
 978         struct bnxt *bp = bnapi->bp;
 979         struct pci_dev *pdev = bp->pdev;
 980         struct sk_buff *skb;
 981
 982         skb = napi_alloc_skb(&bnapi->napi, len);
 983         if (!skb)
 984                 return NULL;
 985
 986         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
 987                                 bp->rx_dir);
 988
 989         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
 990                len + NET_IP_ALIGN);
 991
 992         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
 993                                    bp->rx_dir);
 994
 995         skb_put(skb, len);
 996         return skb;
 997 }
 998
 999 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_napi *bnapi,
1000                            u32 *raw_cons, void *cmp)
1001 {
1002         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1003         struct rx_cmp *rxcmp = cmp;
1004         u32 tmp_raw_cons = *raw_cons;
1005         u8 cmp_type, agg_bufs = 0;
1006
1007         cmp_type = RX_CMP_TYPE(rxcmp);
1008
1009         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1010                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1011                             RX_CMP_AGG_BUFS) >>
1012                            RX_CMP_AGG_BUFS_SHIFT;
1013         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1014                 struct rx_tpa_end_cmp *tpa_end = cmp;
1015
1016                 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1017                             RX_TPA_END_CMP_AGG_BUFS) >>
1018                            RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1019         }
1020
1021         if (agg_bufs) {
1022                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1023                         return -EBUSY;
1024         }
1025         *raw_cons = tmp_raw_cons;
1026         return 0;
1027 }
1028
1029 static void bnxt_queue_sp_work(struct bnxt *bp)
1030 {
1031         if (BNXT_PF(bp))
1032                 queue_work(bnxt_pf_wq, &bp->sp_task);
1033         else
1034                 schedule_work(&bp->sp_task);
1035 }
1036
1037 static void bnxt_cancel_sp_work(struct bnxt *bp)
1038 {
1039         if (BNXT_PF(bp))
1040                 flush_workqueue(bnxt_pf_wq);
1041         else
1042                 cancel_work_sync(&bp->sp_task);
1043 }
1044
1045 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1046 {
1047         if (!rxr->bnapi->in_reset) {
1048                 rxr->bnapi->in_reset = true;
1049                 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1050                 bnxt_queue_sp_work(bp);
1051         }
1052         rxr->rx_next_cons = 0xffff;
1053 }
1054
1055 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1056                            struct rx_tpa_start_cmp *tpa_start,
1057                            struct rx_tpa_start_cmp_ext *tpa_start1)
1058 {
1059         u8 agg_id = TPA_START_AGG_ID(tpa_start);
1060         u16 cons, prod;
1061         struct bnxt_tpa_info *tpa_info;
1062         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1063         struct rx_bd *prod_bd;
1064         dma_addr_t mapping;
1065
1066         cons = tpa_start->rx_tpa_start_cmp_opaque;
1067         prod = rxr->rx_prod;
1068         cons_rx_buf = &rxr->rx_buf_ring[cons];
1069         prod_rx_buf = &rxr->rx_buf_ring[prod];
1070         tpa_info = &rxr->rx_tpa[agg_id];
1071
1072         if (unlikely(cons != rxr->rx_next_cons)) {
1073                 bnxt_sched_reset(bp, rxr);
1074                 return;
1075         }
1076         /* Store cfa_code in tpa_info to use in tpa_end
1077          * completion processing.
1078          */
1079         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1080         prod_rx_buf->data = tpa_info->data;
1081         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1082
1083         mapping = tpa_info->mapping;
1084         prod_rx_buf->mapping = mapping;
1085
1086         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1087
1088         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1089
1090         tpa_info->data = cons_rx_buf->data;
1091         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1092         cons_rx_buf->data = NULL;
1093         tpa_info->mapping = cons_rx_buf->mapping;
1094
1095         tpa_info->len =
1096                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1097                                 RX_TPA_START_CMP_LEN_SHIFT;
1098         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1099                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1100
1101                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1102                 tpa_info->gso_type = SKB_GSO_TCPV4;
1103                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1104                 if (hash_type == 3)
1105                         tpa_info->gso_type = SKB_GSO_TCPV6;
1106                 tpa_info->rss_hash =
1107                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1108         } else {
1109                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1110                 tpa_info->gso_type = 0;
1111                 if (netif_msg_rx_err(bp))
1112                         netdev_warn(bp->dev, "TPA packet without valid hash\n");
1113         }
1114         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1115         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1116         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1117
1118         rxr->rx_prod = NEXT_RX(prod);
1119         cons = NEXT_RX(cons);
1120         rxr->rx_next_cons = NEXT_RX(cons);
1121         cons_rx_buf = &rxr->rx_buf_ring[cons];
1122
1123         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1124         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1125         cons_rx_buf->data = NULL;
1126 }
1127
1128 static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi,
1129                            u16 cp_cons, u32 agg_bufs)
1130 {
1131         if (agg_bufs)
1132                 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
1133 }
1134
1135 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1136                                            int payload_off, int tcp_ts,
1137                                            struct sk_buff *skb)
1138 {
1139 #ifdef CONFIG_INET
1140         struct tcphdr *th;
1141         int len, nw_off;
1142         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1143         u32 hdr_info = tpa_info->hdr_info;
1144         bool loopback = false;
1145
1146         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1147         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1148         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1149
1150         /* If the packet is an internal loopback packet, the offsets will
1151          * have an extra 4 bytes.
1152          */
1153         if (inner_mac_off == 4) {
1154                 loopback = true;
1155         } else if (inner_mac_off > 4) {
1156                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1157                                             ETH_HLEN - 2));
1158
1159                 /* We only support inner iPv4/ipv6.  If we don't see the
1160                  * correct protocol ID, it must be a loopback packet where
1161                  * the offsets are off by 4.
1162                  */
1163                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1164                         loopback = true;
1165         }
1166         if (loopback) {
1167                 /* internal loopback packet, subtract all offsets by 4 */
1168                 inner_ip_off -= 4;
1169                 inner_mac_off -= 4;
1170                 outer_ip_off -= 4;
1171         }
1172
1173         nw_off = inner_ip_off - ETH_HLEN;
1174         skb_set_network_header(skb, nw_off);
1175         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1176                 struct ipv6hdr *iph = ipv6_hdr(skb);
1177
1178                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1179                 len = skb->len - skb_transport_offset(skb);
1180                 th = tcp_hdr(skb);
1181                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1182         } else {
1183                 struct iphdr *iph = ip_hdr(skb);
1184
1185                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1186                 len = skb->len - skb_transport_offset(skb);
1187                 th = tcp_hdr(skb);
1188                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1189         }
1190
1191         if (inner_mac_off) { /* tunnel */
1192                 struct udphdr *uh = NULL;
1193                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1194                                             ETH_HLEN - 2));
1195
1196                 if (proto == htons(ETH_P_IP)) {
1197                         struct iphdr *iph = (struct iphdr *)skb->data;
1198
1199                         if (iph->protocol == IPPROTO_UDP)
1200                                 uh = (struct udphdr *)(iph + 1);
1201                 } else {
1202                         struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1203
1204                         if (iph->nexthdr == IPPROTO_UDP)
1205                                 uh = (struct udphdr *)(iph + 1);
1206                 }
1207                 if (uh) {
1208                         if (uh->check)
1209                                 skb_shinfo(skb)->gso_type |=
1210                                         SKB_GSO_UDP_TUNNEL_CSUM;
1211                         else
1212                                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1213                 }
1214         }
1215 #endif
1216         return skb;
1217 }
1218
1219 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1220 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1221
1222 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1223                                            int payload_off, int tcp_ts,
1224                                            struct sk_buff *skb)
1225 {
1226 #ifdef CONFIG_INET
1227         struct tcphdr *th;
1228         int len, nw_off, tcp_opt_len = 0;
1229
1230         if (tcp_ts)
1231                 tcp_opt_len = 12;
1232
1233         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1234                 struct iphdr *iph;
1235
1236                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1237                          ETH_HLEN;
1238                 skb_set_network_header(skb, nw_off);
1239                 iph = ip_hdr(skb);
1240                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1241                 len = skb->len - skb_transport_offset(skb);
1242                 th = tcp_hdr(skb);
1243                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1244         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1245                 struct ipv6hdr *iph;
1246
1247                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1248                          ETH_HLEN;
1249                 skb_set_network_header(skb, nw_off);
1250                 iph = ipv6_hdr(skb);
1251                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1252                 len = skb->len - skb_transport_offset(skb);
1253                 th = tcp_hdr(skb);
1254                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1255         } else {
1256                 dev_kfree_skb_any(skb);
1257                 return NULL;
1258         }
1259
1260         if (nw_off) { /* tunnel */
1261                 struct udphdr *uh = NULL;
1262
1263                 if (skb->protocol == htons(ETH_P_IP)) {
1264                         struct iphdr *iph = (struct iphdr *)skb->data;
1265
1266                         if (iph->protocol == IPPROTO_UDP)
1267                                 uh = (struct udphdr *)(iph + 1);
1268                 } else {
1269                         struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1270
1271                         if (iph->nexthdr == IPPROTO_UDP)
1272                                 uh = (struct udphdr *)(iph + 1);
1273                 }
1274                 if (uh) {
1275                         if (uh->check)
1276                                 skb_shinfo(skb)->gso_type |=
1277                                         SKB_GSO_UDP_TUNNEL_CSUM;
1278                         else
1279                                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1280                 }
1281         }
1282 #endif
1283         return skb;
1284 }
1285
1286 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1287                                            struct bnxt_tpa_info *tpa_info,
1288                                            struct rx_tpa_end_cmp *tpa_end,
1289                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1290                                            struct sk_buff *skb)
1291 {
1292 #ifdef CONFIG_INET
1293         int payload_off;
1294         u16 segs;
1295
1296         segs = TPA_END_TPA_SEGS(tpa_end);
1297         if (segs == 1)
1298                 return skb;
1299
1300         NAPI_GRO_CB(skb)->count = segs;
1301         skb_shinfo(skb)->gso_size =
1302                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1303         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1304         payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1305                        RX_TPA_END_CMP_PAYLOAD_OFFSET) >>
1306                       RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT;
1307         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1308         if (likely(skb))
1309                 tcp_gro_complete(skb);
1310 #endif
1311         return skb;
1312 }
1313
1314 /* Given the cfa_code of a received packet determine which
1315  * netdev (vf-rep or PF) the packet is destined to.
1316  */
1317 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1318 {
1319         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1320
1321         /* if vf-rep dev is NULL, the must belongs to the PF */
1322         return dev ? dev : bp->dev;
1323 }
1324
1325 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1326                                            struct bnxt_napi *bnapi,
1327                                            u32 *raw_cons,
1328                                            struct rx_tpa_end_cmp *tpa_end,
1329                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1330                                            u8 *event)
1331 {
1332         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1333         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1334         u8 agg_id = TPA_END_AGG_ID(tpa_end);
1335         u8 *data_ptr, agg_bufs;
1336         u16 cp_cons = RING_CMP(*raw_cons);
1337         unsigned int len;
1338         struct bnxt_tpa_info *tpa_info;
1339         dma_addr_t mapping;
1340         struct sk_buff *skb;
1341         void *data;
1342
1343         if (unlikely(bnapi->in_reset)) {
1344                 int rc = bnxt_discard_rx(bp, bnapi, raw_cons, tpa_end);
1345
1346                 if (rc < 0)
1347                         return ERR_PTR(-EBUSY);
1348                 return NULL;
1349         }
1350
1351         tpa_info = &rxr->rx_tpa[agg_id];
1352         data = tpa_info->data;
1353         data_ptr = tpa_info->data_ptr;
1354         prefetch(data_ptr);
1355         len = tpa_info->len;
1356         mapping = tpa_info->mapping;
1357
1358         agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1359                     RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1360
1361         if (agg_bufs) {
1362                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1363                         return ERR_PTR(-EBUSY);
1364
1365                 *event |= BNXT_AGG_EVENT;
1366                 cp_cons = NEXT_CMP(cp_cons);
1367         }
1368
1369         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1370                 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1371                 if (agg_bufs > MAX_SKB_FRAGS)
1372                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1373                                     agg_bufs, (int)MAX_SKB_FRAGS);
1374                 return NULL;
1375         }
1376
1377         if (len <= bp->rx_copy_thresh) {
1378                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1379                 if (!skb) {
1380                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1381                         return NULL;
1382                 }
1383         } else {
1384                 u8 *new_data;
1385                 dma_addr_t new_mapping;
1386
1387                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1388                 if (!new_data) {
1389                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1390                         return NULL;
1391                 }
1392
1393                 tpa_info->data = new_data;
1394                 tpa_info->data_ptr = new_data + bp->rx_offset;
1395                 tpa_info->mapping = new_mapping;
1396
1397                 skb = build_skb(data, 0);
1398                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1399                                        bp->rx_buf_use_size, bp->rx_dir,
1400                                        DMA_ATTR_WEAK_ORDERING);
1401
1402                 if (!skb) {
1403                         kfree(data);
1404                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1405                         return NULL;
1406                 }
1407                 skb_reserve(skb, bp->rx_offset);
1408                 skb_put(skb, len);
1409         }
1410
1411         if (agg_bufs) {
1412                 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1413                 if (!skb) {
1414                         /* Page reuse already handled by bnxt_rx_pages(). */
1415                         return NULL;
1416                 }
1417         }
1418
1419         skb->protocol =
1420                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1421
1422         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1423                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1424
1425         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1426             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1427                 u16 vlan_proto = tpa_info->metadata >>
1428                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1429                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_VID_MASK;
1430
1431                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1432         }
1433
1434         skb_checksum_none_assert(skb);
1435         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1436                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1437                 skb->csum_level =
1438                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1439         }
1440
1441         if (TPA_END_GRO(tpa_end))
1442                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1443
1444         return skb;
1445 }
1446
1447 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1448                              struct sk_buff *skb)
1449 {
1450         if (skb->dev != bp->dev) {
1451                 /* this packet belongs to a vf-rep */
1452                 bnxt_vf_rep_rx(bp, skb);
1453                 return;
1454         }
1455         skb_record_rx_queue(skb, bnapi->index);
1456         napi_gro_receive(&bnapi->napi, skb);
1457 }
1458
1459 /* returns the following:
1460  * 1       - 1 packet successfully received
1461  * 0       - successful TPA_START, packet not completed yet
1462  * -EBUSY  - completion ring does not have all the agg buffers yet
1463  * -ENOMEM - packet aborted due to out of memory
1464  * -EIO    - packet aborted due to hw error indicated in BD
1465  */
1466 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
1467                        u8 *event)
1468 {
1469         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1470         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1471         struct net_device *dev = bp->dev;
1472         struct rx_cmp *rxcmp;
1473         struct rx_cmp_ext *rxcmp1;
1474         u32 tmp_raw_cons = *raw_cons;
1475         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1476         struct bnxt_sw_rx_bd *rx_buf;
1477         unsigned int len;
1478         u8 *data_ptr, agg_bufs, cmp_type;
1479         dma_addr_t dma_addr;
1480         struct sk_buff *skb;
1481         void *data;
1482         int rc = 0;
1483         u32 misc;
1484
1485         rxcmp = (struct rx_cmp *)
1486                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1487
1488         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1489         cp_cons = RING_CMP(tmp_raw_cons);
1490         rxcmp1 = (struct rx_cmp_ext *)
1491                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1492
1493         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1494                 return -EBUSY;
1495
1496         cmp_type = RX_CMP_TYPE(rxcmp);
1497
1498         prod = rxr->rx_prod;
1499
1500         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1501                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1502                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1503
1504                 *event |= BNXT_RX_EVENT;
1505                 goto next_rx_no_prod;
1506
1507         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1508                 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons,
1509                                    (struct rx_tpa_end_cmp *)rxcmp,
1510                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1511
1512                 if (unlikely(IS_ERR(skb)))
1513                         return -EBUSY;
1514
1515                 rc = -ENOMEM;
1516                 if (likely(skb)) {
1517                         bnxt_deliver_skb(bp, bnapi, skb);
1518                         rc = 1;
1519                 }
1520                 *event |= BNXT_RX_EVENT;
1521                 goto next_rx_no_prod;
1522         }
1523
1524         cons = rxcmp->rx_cmp_opaque;
1525         rx_buf = &rxr->rx_buf_ring[cons];
1526         data = rx_buf->data;
1527         data_ptr = rx_buf->data_ptr;
1528         if (unlikely(cons != rxr->rx_next_cons)) {
1529                 int rc1 = bnxt_discard_rx(bp, bnapi, raw_cons, rxcmp);
1530
1531                 bnxt_sched_reset(bp, rxr);
1532                 return rc1;
1533         }
1534         prefetch(data_ptr);
1535
1536         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1537         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1538
1539         if (agg_bufs) {
1540                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1541                         return -EBUSY;
1542
1543                 cp_cons = NEXT_CMP(cp_cons);
1544                 *event |= BNXT_AGG_EVENT;
1545         }
1546         *event |= BNXT_RX_EVENT;
1547
1548         rx_buf->data = NULL;
1549         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1550                 bnxt_reuse_rx_data(rxr, cons, data);
1551                 if (agg_bufs)
1552                         bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
1553
1554                 rc = -EIO;
1555                 goto next_rx;
1556         }
1557
1558         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1559         dma_addr = rx_buf->mapping;
1560
1561         if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1562                 rc = 1;
1563                 goto next_rx;
1564         }
1565
1566         if (len <= bp->rx_copy_thresh) {
1567                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1568                 bnxt_reuse_rx_data(rxr, cons, data);
1569                 if (!skb) {
1570                         rc = -ENOMEM;
1571                         goto next_rx;
1572                 }
1573         } else {
1574                 u32 payload;
1575
1576                 if (rx_buf->data_ptr == data_ptr)
1577                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1578                 else
1579                         payload = 0;
1580                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1581                                       payload | len);
1582                 if (!skb) {
1583                         rc = -ENOMEM;
1584                         goto next_rx;
1585                 }
1586         }
1587
1588         if (agg_bufs) {
1589                 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1590                 if (!skb) {
1591                         rc = -ENOMEM;
1592                         goto next_rx;
1593                 }
1594         }
1595
1596         if (RX_CMP_HASH_VALID(rxcmp)) {
1597                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1598                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1599
1600                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1601                 if (hash_type != 1 && hash_type != 3)
1602                         type = PKT_HASH_TYPE_L3;
1603                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1604         }
1605
1606         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1607         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1608
1609         if ((rxcmp1->rx_cmp_flags2 &
1610              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1611             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1612                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1613                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_VID_MASK;
1614                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1615
1616                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1617         }
1618
1619         skb_checksum_none_assert(skb);
1620         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1621                 if (dev->features & NETIF_F_RXCSUM) {
1622                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1623                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1624                 }
1625         } else {
1626                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1627                         if (dev->features & NETIF_F_RXCSUM)
1628                                 cpr->rx_l4_csum_errors++;
1629                 }
1630         }
1631
1632         bnxt_deliver_skb(bp, bnapi, skb);
1633         rc = 1;
1634
1635 next_rx:
1636         rxr->rx_prod = NEXT_RX(prod);
1637         rxr->rx_next_cons = NEXT_RX(cons);
1638
1639 next_rx_no_prod:
1640         *raw_cons = tmp_raw_cons;
1641
1642         return rc;
1643 }
1644
1645 /* In netpoll mode, if we are using a combined completion ring, we need to
1646  * discard the rx packets and recycle the buffers.
1647  */
1648 static int bnxt_force_rx_discard(struct bnxt *bp, struct bnxt_napi *bnapi,
1649                                  u32 *raw_cons, u8 *event)
1650 {
1651         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1652         u32 tmp_raw_cons = *raw_cons;
1653         struct rx_cmp_ext *rxcmp1;
1654         struct rx_cmp *rxcmp;
1655         u16 cp_cons;
1656         u8 cmp_type;
1657
1658         cp_cons = RING_CMP(tmp_raw_cons);
1659         rxcmp = (struct rx_cmp *)
1660                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1661
1662         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1663         cp_cons = RING_CMP(tmp_raw_cons);
1664         rxcmp1 = (struct rx_cmp_ext *)
1665                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1666
1667         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1668                 return -EBUSY;
1669
1670         cmp_type = RX_CMP_TYPE(rxcmp);
1671         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1672                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1673                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1674         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1675                 struct rx_tpa_end_cmp_ext *tpa_end1;
1676
1677                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1678                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1679                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1680         }
1681         return bnxt_rx_pkt(bp, bnapi, raw_cons, event);
1682 }
1683
1684 #define BNXT_GET_EVENT_PORT(data)       \
1685         ((data) &                       \
1686          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1687
1688 static int bnxt_async_event_process(struct bnxt *bp,
1689                                     struct hwrm_async_event_cmpl *cmpl)
1690 {
1691         u16 event_id = le16_to_cpu(cmpl->event_id);
1692
1693         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1694         switch (event_id) {
1695         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1696                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1697                 struct bnxt_link_info *link_info = &bp->link_info;
1698
1699                 if (BNXT_VF(bp))
1700                         goto async_event_process_exit;
1701                 if (data1 & 0x20000) {
1702                         u16 fw_speed = link_info->force_link_speed;
1703                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1704
1705                         netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1706                                     speed);
1707                 }
1708                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1709                 /* fall thru */
1710         }
1711         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1712                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1713                 break;
1714         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1715                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1716                 break;
1717         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1718                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1719                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1720
1721                 if (BNXT_VF(bp))
1722                         break;
1723
1724                 if (bp->pf.port_id != port_id)
1725                         break;
1726
1727                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1728                 break;
1729         }
1730         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
1731                 if (BNXT_PF(bp))
1732                         goto async_event_process_exit;
1733                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
1734                 break;
1735         default:
1736                 goto async_event_process_exit;
1737         }
1738         bnxt_queue_sp_work(bp);
1739 async_event_process_exit:
1740         bnxt_ulp_async_events(bp, cmpl);
1741         return 0;
1742 }
1743
1744 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1745 {
1746         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1747         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1748         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1749                                 (struct hwrm_fwd_req_cmpl *)txcmp;
1750
1751         switch (cmpl_type) {
1752         case CMPL_BASE_TYPE_HWRM_DONE:
1753                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1754                 if (seq_id == bp->hwrm_intr_seq_id)
1755                         bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID;
1756                 else
1757                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1758                 break;
1759
1760         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1761                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1762
1763                 if ((vf_id < bp->pf.first_vf_id) ||
1764                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1765                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1766                                    vf_id);
1767                         return -EINVAL;
1768                 }
1769
1770                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1771                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
1772                 bnxt_queue_sp_work(bp);
1773                 break;
1774
1775         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1776                 bnxt_async_event_process(bp,
1777                                          (struct hwrm_async_event_cmpl *)txcmp);
1778
1779         default:
1780                 break;
1781         }
1782
1783         return 0;
1784 }
1785
1786 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
1787 {
1788         struct bnxt_napi *bnapi = dev_instance;
1789         struct bnxt *bp = bnapi->bp;
1790         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1791         u32 cons = RING_CMP(cpr->cp_raw_cons);
1792
1793         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1794         napi_schedule(&bnapi->napi);
1795         return IRQ_HANDLED;
1796 }
1797
1798 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
1799 {
1800         u32 raw_cons = cpr->cp_raw_cons;
1801         u16 cons = RING_CMP(raw_cons);
1802         struct tx_cmp *txcmp;
1803
1804         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1805
1806         return TX_CMP_VALID(txcmp, raw_cons);
1807 }
1808
1809 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
1810 {
1811         struct bnxt_napi *bnapi = dev_instance;
1812         struct bnxt *bp = bnapi->bp;
1813         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1814         u32 cons = RING_CMP(cpr->cp_raw_cons);
1815         u32 int_status;
1816
1817         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1818
1819         if (!bnxt_has_work(bp, cpr)) {
1820                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
1821                 /* return if erroneous interrupt */
1822                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
1823                         return IRQ_NONE;
1824         }
1825
1826         /* disable ring IRQ */
1827         BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell);
1828
1829         /* Return here if interrupt is shared and is disabled. */
1830         if (unlikely(atomic_read(&bp->intr_sem) != 0))
1831                 return IRQ_HANDLED;
1832
1833         napi_schedule(&bnapi->napi);
1834         return IRQ_HANDLED;
1835 }
1836
1837 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
1838 {
1839         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1840         u32 raw_cons = cpr->cp_raw_cons;
1841         u32 cons;
1842         int tx_pkts = 0;
1843         int rx_pkts = 0;
1844         u8 event = 0;
1845         struct tx_cmp *txcmp;
1846
1847         while (1) {
1848                 int rc;
1849
1850                 cons = RING_CMP(raw_cons);
1851                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1852
1853                 if (!TX_CMP_VALID(txcmp, raw_cons))
1854                         break;
1855
1856                 /* The valid test of the entry must be done first before
1857                  * reading any further.
1858                  */
1859                 dma_rmb();
1860                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
1861                         tx_pkts++;
1862                         /* return full budget so NAPI will complete. */
1863                         if (unlikely(tx_pkts > bp->tx_wake_thresh))
1864                                 rx_pkts = budget;
1865                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1866                         if (likely(budget))
1867                                 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event);
1868                         else
1869                                 rc = bnxt_force_rx_discard(bp, bnapi, &raw_cons,
1870                                                            &event);
1871                         if (likely(rc >= 0))
1872                                 rx_pkts += rc;
1873                         /* Increment rx_pkts when rc is -ENOMEM to count towards
1874                          * the NAPI budget.  Otherwise, we may potentially loop
1875                          * here forever if we consistently cannot allocate
1876                          * buffers.
1877                          */
1878                         else if (rc == -ENOMEM)
1879                                 rx_pkts++;
1880                         else if (rc == -EBUSY)  /* partial completion */
1881                                 break;
1882                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
1883                                      CMPL_BASE_TYPE_HWRM_DONE) ||
1884                                     (TX_CMP_TYPE(txcmp) ==
1885                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
1886                                     (TX_CMP_TYPE(txcmp) ==
1887                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
1888                         bnxt_hwrm_handler(bp, txcmp);
1889                 }
1890                 raw_cons = NEXT_RAW_CMP(raw_cons);
1891
1892                 if (rx_pkts == budget)
1893                         break;
1894         }
1895
1896         if (event & BNXT_TX_EVENT) {
1897                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
1898                 void __iomem *db = txr->tx_doorbell;
1899                 u16 prod = txr->tx_prod;
1900
1901                 /* Sync BD data before updating doorbell */
1902                 wmb();
1903
1904                 bnxt_db_write(bp, db, DB_KEY_TX | prod);
1905         }
1906
1907         cpr->cp_raw_cons = raw_cons;
1908         /* ACK completion ring before freeing tx ring and producing new
1909          * buffers in rx/agg rings to prevent overflowing the completion
1910          * ring.
1911          */
1912         BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
1913
1914         if (tx_pkts)
1915                 bnapi->tx_int(bp, bnapi, tx_pkts);
1916
1917         if (event & BNXT_RX_EVENT) {
1918                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1919
1920                 bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod);
1921                 if (event & BNXT_AGG_EVENT)
1922                         bnxt_db_write(bp, rxr->rx_agg_doorbell,
1923                                       DB_KEY_RX | rxr->rx_agg_prod);
1924         }
1925         return rx_pkts;
1926 }
1927
1928 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
1929 {
1930         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1931         struct bnxt *bp = bnapi->bp;
1932         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1933         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1934         struct tx_cmp *txcmp;
1935         struct rx_cmp_ext *rxcmp1;
1936         u32 cp_cons, tmp_raw_cons;
1937         u32 raw_cons = cpr->cp_raw_cons;
1938         u32 rx_pkts = 0;
1939         u8 event = 0;
1940
1941         while (1) {
1942                 int rc;
1943
1944                 cp_cons = RING_CMP(raw_cons);
1945                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1946
1947                 if (!TX_CMP_VALID(txcmp, raw_cons))
1948                         break;
1949
1950                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1951                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
1952                         cp_cons = RING_CMP(tmp_raw_cons);
1953                         rxcmp1 = (struct rx_cmp_ext *)
1954                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1955
1956                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1957                                 break;
1958
1959                         /* force an error to recycle the buffer */
1960                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1961                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1962
1963                         rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event);
1964                         if (likely(rc == -EIO))
1965                                 rx_pkts++;
1966                         else if (rc == -EBUSY)  /* partial completion */
1967                                 break;
1968                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
1969                                     CMPL_BASE_TYPE_HWRM_DONE)) {
1970                         bnxt_hwrm_handler(bp, txcmp);
1971                 } else {
1972                         netdev_err(bp->dev,
1973                                    "Invalid completion received on special ring\n");
1974                 }
1975                 raw_cons = NEXT_RAW_CMP(raw_cons);
1976
1977                 if (rx_pkts == budget)
1978                         break;
1979         }
1980
1981         cpr->cp_raw_cons = raw_cons;
1982         BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
1983         bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod);
1984
1985         if (event & BNXT_AGG_EVENT)
1986                 bnxt_db_write(bp, rxr->rx_agg_doorbell,
1987                               DB_KEY_RX | rxr->rx_agg_prod);
1988
1989         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
1990                 napi_complete_done(napi, rx_pkts);
1991                 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
1992         }
1993         return rx_pkts;
1994 }
1995
1996 static int bnxt_poll(struct napi_struct *napi, int budget)
1997 {
1998         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1999         struct bnxt *bp = bnapi->bp;
2000         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2001         int work_done = 0;
2002
2003         while (1) {
2004                 work_done += bnxt_poll_work(bp, bnapi, budget - work_done);
2005
2006                 if (work_done >= budget)
2007                         break;
2008
2009                 if (!bnxt_has_work(bp, cpr)) {
2010                         if (napi_complete_done(napi, work_done))
2011                                 BNXT_CP_DB_REARM(cpr->cp_doorbell,
2012                                                  cpr->cp_raw_cons);
2013                         break;
2014                 }
2015         }
2016         mmiowb();
2017         return work_done;
2018 }
2019
2020 static void bnxt_free_tx_skbs(struct bnxt *bp)
2021 {
2022         int i, max_idx;
2023         struct pci_dev *pdev = bp->pdev;
2024
2025         if (!bp->tx_ring)
2026                 return;
2027
2028         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2029         for (i = 0; i < bp->tx_nr_rings; i++) {
2030                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2031                 int j;
2032
2033                 for (j = 0; j < max_idx;) {
2034                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2035                         struct sk_buff *skb = tx_buf->skb;
2036                         int k, last;
2037
2038                         if (!skb) {
2039                                 j++;
2040                                 continue;
2041                         }
2042
2043                         tx_buf->skb = NULL;
2044
2045                         if (tx_buf->is_push) {
2046                                 dev_kfree_skb(skb);
2047                                 j += 2;
2048                                 continue;
2049                         }
2050
2051                         dma_unmap_single(&pdev->dev,
2052                                          dma_unmap_addr(tx_buf, mapping),
2053                                          skb_headlen(skb),
2054                                          PCI_DMA_TODEVICE);
2055
2056                         last = tx_buf->nr_frags;
2057                         j += 2;
2058                         for (k = 0; k < last; k++, j++) {
2059                                 int ring_idx = j & bp->tx_ring_mask;
2060                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2061
2062                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2063                                 dma_unmap_page(
2064                                         &pdev->dev,
2065                                         dma_unmap_addr(tx_buf, mapping),
2066                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
2067                         }
2068                         dev_kfree_skb(skb);
2069                 }
2070                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2071         }
2072 }
2073
2074 static void bnxt_free_rx_skbs(struct bnxt *bp)
2075 {
2076         int i, max_idx, max_agg_idx;
2077         struct pci_dev *pdev = bp->pdev;
2078
2079         if (!bp->rx_ring)
2080                 return;
2081
2082         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2083         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2084         for (i = 0; i < bp->rx_nr_rings; i++) {
2085                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2086                 int j;
2087
2088                 if (rxr->rx_tpa) {
2089                         for (j = 0; j < MAX_TPA; j++) {
2090                                 struct bnxt_tpa_info *tpa_info =
2091                                                         &rxr->rx_tpa[j];
2092                                 u8 *data = tpa_info->data;
2093
2094                                 if (!data)
2095                                         continue;
2096
2097                                 dma_unmap_single_attrs(&pdev->dev,
2098                                                        tpa_info->mapping,
2099                                                        bp->rx_buf_use_size,
2100                                                        bp->rx_dir,
2101                                                        DMA_ATTR_WEAK_ORDERING);
2102
2103                                 tpa_info->data = NULL;
2104
2105                                 kfree(data);
2106                         }
2107                 }
2108
2109                 for (j = 0; j < max_idx; j++) {
2110                         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
2111                         dma_addr_t mapping = rx_buf->mapping;
2112                         void *data = rx_buf->data;
2113
2114                         if (!data)
2115                                 continue;
2116
2117                         rx_buf->data = NULL;
2118
2119                         if (BNXT_RX_PAGE_MODE(bp)) {
2120                                 mapping -= bp->rx_dma_offset;
2121                                 dma_unmap_page_attrs(&pdev->dev, mapping,
2122                                                      PAGE_SIZE, bp->rx_dir,
2123                                                      DMA_ATTR_WEAK_ORDERING);
2124                                 __free_page(data);
2125                         } else {
2126                                 dma_unmap_single_attrs(&pdev->dev, mapping,
2127                                                        bp->rx_buf_use_size,
2128                                                        bp->rx_dir,
2129                                                        DMA_ATTR_WEAK_ORDERING);
2130                                 kfree(data);
2131                         }
2132                 }
2133
2134                 for (j = 0; j < max_agg_idx; j++) {
2135                         struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2136                                 &rxr->rx_agg_ring[j];
2137                         struct page *page = rx_agg_buf->page;
2138
2139                         if (!page)
2140                                 continue;
2141
2142                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2143                                              BNXT_RX_PAGE_SIZE,
2144                                              PCI_DMA_FROMDEVICE,
2145                                              DMA_ATTR_WEAK_ORDERING);
2146
2147                         rx_agg_buf->page = NULL;
2148                         __clear_bit(j, rxr->rx_agg_bmap);
2149
2150                         __free_page(page);
2151                 }
2152                 if (rxr->rx_page) {
2153                         __free_page(rxr->rx_page);
2154                         rxr->rx_page = NULL;
2155                 }
2156         }
2157 }
2158
2159 static void bnxt_free_skbs(struct bnxt *bp)
2160 {
2161         bnxt_free_tx_skbs(bp);
2162         bnxt_free_rx_skbs(bp);
2163 }
2164
2165 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
2166 {
2167         struct pci_dev *pdev = bp->pdev;
2168         int i;
2169
2170         for (i = 0; i < ring->nr_pages; i++) {
2171                 if (!ring->pg_arr[i])
2172                         continue;
2173
2174                 dma_free_coherent(&pdev->dev, ring->page_size,
2175                                   ring->pg_arr[i], ring->dma_arr[i]);
2176
2177                 ring->pg_arr[i] = NULL;
2178         }
2179         if (ring->pg_tbl) {
2180                 dma_free_coherent(&pdev->dev, ring->nr_pages * 8,
2181                                   ring->pg_tbl, ring->pg_tbl_map);
2182                 ring->pg_tbl = NULL;
2183         }
2184         if (ring->vmem_size && *ring->vmem) {
2185                 vfree(*ring->vmem);
2186                 *ring->vmem = NULL;
2187         }
2188 }
2189
2190 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
2191 {
2192         int i;
2193         struct pci_dev *pdev = bp->pdev;
2194
2195         if (ring->nr_pages > 1) {
2196                 ring->pg_tbl = dma_alloc_coherent(&pdev->dev,
2197                                                   ring->nr_pages * 8,
2198                                                   &ring->pg_tbl_map,
2199                                                   GFP_KERNEL);
2200                 if (!ring->pg_tbl)
2201                         return -ENOMEM;
2202         }
2203
2204         for (i = 0; i < ring->nr_pages; i++) {
2205                 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2206                                                      ring->page_size,
2207                                                      &ring->dma_arr[i],
2208                                                      GFP_KERNEL);
2209                 if (!ring->pg_arr[i])
2210                         return -ENOMEM;
2211
2212                 if (ring->nr_pages > 1)
2213                         ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]);
2214         }
2215
2216         if (ring->vmem_size) {
2217                 *ring->vmem = vzalloc(ring->vmem_size);
2218                 if (!(*ring->vmem))
2219                         return -ENOMEM;
2220         }
2221         return 0;
2222 }
2223
2224 static void bnxt_free_rx_rings(struct bnxt *bp)
2225 {
2226         int i;
2227
2228         if (!bp->rx_ring)
2229                 return;
2230
2231         for (i = 0; i < bp->rx_nr_rings; i++) {
2232                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2233                 struct bnxt_ring_struct *ring;
2234
2235                 if (rxr->xdp_prog)
2236                         bpf_prog_put(rxr->xdp_prog);
2237
2238                 kfree(rxr->rx_tpa);
2239                 rxr->rx_tpa = NULL;
2240
2241                 kfree(rxr->rx_agg_bmap);
2242                 rxr->rx_agg_bmap = NULL;
2243
2244                 ring = &rxr->rx_ring_struct;
2245                 bnxt_free_ring(bp, ring);
2246
2247                 ring = &rxr->rx_agg_ring_struct;
2248                 bnxt_free_ring(bp, ring);
2249         }
2250 }
2251
2252 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2253 {
2254         int i, rc, agg_rings = 0, tpa_rings = 0;
2255
2256         if (!bp->rx_ring)
2257                 return -ENOMEM;
2258
2259         if (bp->flags & BNXT_FLAG_AGG_RINGS)
2260                 agg_rings = 1;
2261
2262         if (bp->flags & BNXT_FLAG_TPA)
2263                 tpa_rings = 1;
2264
2265         for (i = 0; i < bp->rx_nr_rings; i++) {
2266                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2267                 struct bnxt_ring_struct *ring;
2268
2269                 ring = &rxr->rx_ring_struct;
2270
2271                 rc = bnxt_alloc_ring(bp, ring);
2272                 if (rc)
2273                         return rc;
2274
2275                 if (agg_rings) {
2276                         u16 mem_size;
2277
2278                         ring = &rxr->rx_agg_ring_struct;
2279                         rc = bnxt_alloc_ring(bp, ring);
2280                         if (rc)
2281                                 return rc;
2282
2283                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2284                         mem_size = rxr->rx_agg_bmap_size / 8;
2285                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2286                         if (!rxr->rx_agg_bmap)
2287                                 return -ENOMEM;
2288
2289                         if (tpa_rings) {
2290                                 rxr->rx_tpa = kcalloc(MAX_TPA,
2291                                                 sizeof(struct bnxt_tpa_info),
2292                                                 GFP_KERNEL);
2293                                 if (!rxr->rx_tpa)
2294                                         return -ENOMEM;
2295                         }
2296                 }
2297         }
2298         return 0;
2299 }
2300
2301 static void bnxt_free_tx_rings(struct bnxt *bp)
2302 {
2303         int i;
2304         struct pci_dev *pdev = bp->pdev;
2305
2306         if (!bp->tx_ring)
2307                 return;
2308
2309         for (i = 0; i < bp->tx_nr_rings; i++) {
2310                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2311                 struct bnxt_ring_struct *ring;
2312
2313                 if (txr->tx_push) {
2314                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
2315                                           txr->tx_push, txr->tx_push_mapping);
2316                         txr->tx_push = NULL;
2317                 }
2318
2319                 ring = &txr->tx_ring_struct;
2320
2321                 bnxt_free_ring(bp, ring);
2322         }
2323 }
2324
2325 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2326 {
2327         int i, j, rc;
2328         struct pci_dev *pdev = bp->pdev;
2329
2330         bp->tx_push_size = 0;
2331         if (bp->tx_push_thresh) {
2332                 int push_size;
2333
2334                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2335                                         bp->tx_push_thresh);
2336
2337                 if (push_size > 256) {
2338                         push_size = 0;
2339                         bp->tx_push_thresh = 0;
2340                 }
2341
2342                 bp->tx_push_size = push_size;
2343         }
2344
2345         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2346                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2347                 struct bnxt_ring_struct *ring;
2348
2349                 ring = &txr->tx_ring_struct;
2350
2351                 rc = bnxt_alloc_ring(bp, ring);
2352                 if (rc)
2353                         return rc;
2354
2355                 if (bp->tx_push_size) {
2356                         dma_addr_t mapping;
2357
2358                         /* One pre-allocated DMA buffer to backup
2359                          * TX push operation
2360                          */
2361                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
2362                                                 bp->tx_push_size,
2363                                                 &txr->tx_push_mapping,
2364                                                 GFP_KERNEL);
2365
2366                         if (!txr->tx_push)
2367                                 return -ENOMEM;
2368
2369                         mapping = txr->tx_push_mapping +
2370                                 sizeof(struct tx_push_bd);
2371                         txr->data_mapping = cpu_to_le64(mapping);
2372
2373                         memset(txr->tx_push, 0, sizeof(struct tx_push_bd));
2374                 }
2375                 ring->queue_id = bp->q_info[j].queue_id;
2376                 if (i < bp->tx_nr_rings_xdp)
2377                         continue;
2378                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2379                         j++;
2380         }
2381         return 0;
2382 }
2383
2384 static void bnxt_free_cp_rings(struct bnxt *bp)
2385 {
2386         int i;
2387
2388         if (!bp->bnapi)
2389                 return;
2390
2391         for (i = 0; i < bp->cp_nr_rings; i++) {
2392                 struct bnxt_napi *bnapi = bp->bnapi[i];
2393                 struct bnxt_cp_ring_info *cpr;
2394                 struct bnxt_ring_struct *ring;
2395
2396                 if (!bnapi)
2397                         continue;
2398
2399                 cpr = &bnapi->cp_ring;
2400                 ring = &cpr->cp_ring_struct;
2401
2402                 bnxt_free_ring(bp, ring);
2403         }
2404 }
2405
2406 static int bnxt_alloc_cp_rings(struct bnxt *bp)
2407 {
2408         int i, rc;
2409
2410         for (i = 0; i < bp->cp_nr_rings; i++) {
2411                 struct bnxt_napi *bnapi = bp->bnapi[i];
2412                 struct bnxt_cp_ring_info *cpr;
2413                 struct bnxt_ring_struct *ring;
2414
2415                 if (!bnapi)
2416                         continue;
2417
2418                 cpr = &bnapi->cp_ring;
2419                 ring = &cpr->cp_ring_struct;
2420
2421                 rc = bnxt_alloc_ring(bp, ring);
2422                 if (rc)
2423                         return rc;
2424         }
2425         return 0;
2426 }
2427
2428 static void bnxt_init_ring_struct(struct bnxt *bp)
2429 {
2430         int i;
2431
2432         for (i = 0; i < bp->cp_nr_rings; i++) {
2433                 struct bnxt_napi *bnapi = bp->bnapi[i];
2434                 struct bnxt_cp_ring_info *cpr;
2435                 struct bnxt_rx_ring_info *rxr;
2436                 struct bnxt_tx_ring_info *txr;
2437                 struct bnxt_ring_struct *ring;
2438
2439                 if (!bnapi)
2440                         continue;
2441
2442                 cpr = &bnapi->cp_ring;
2443                 ring = &cpr->cp_ring_struct;
2444                 ring->nr_pages = bp->cp_nr_pages;
2445                 ring->page_size = HW_CMPD_RING_SIZE;
2446                 ring->pg_arr = (void **)cpr->cp_desc_ring;
2447                 ring->dma_arr = cpr->cp_desc_mapping;
2448                 ring->vmem_size = 0;
2449
2450                 rxr = bnapi->rx_ring;
2451                 if (!rxr)
2452                         goto skip_rx;
2453
2454                 ring = &rxr->rx_ring_struct;
2455                 ring->nr_pages = bp->rx_nr_pages;
2456                 ring->page_size = HW_RXBD_RING_SIZE;
2457                 ring->pg_arr = (void **)rxr->rx_desc_ring;
2458                 ring->dma_arr = rxr->rx_desc_mapping;
2459                 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
2460                 ring->vmem = (void **)&rxr->rx_buf_ring;
2461
2462                 ring = &rxr->rx_agg_ring_struct;
2463                 ring->nr_pages = bp->rx_agg_nr_pages;
2464                 ring->page_size = HW_RXBD_RING_SIZE;
2465                 ring->pg_arr = (void **)rxr->rx_agg_desc_ring;
2466                 ring->dma_arr = rxr->rx_agg_desc_mapping;
2467                 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
2468                 ring->vmem = (void **)&rxr->rx_agg_ring;
2469
2470 skip_rx:
2471                 txr = bnapi->tx_ring;
2472                 if (!txr)
2473                         continue;
2474
2475                 ring = &txr->tx_ring_struct;
2476                 ring->nr_pages = bp->tx_nr_pages;
2477                 ring->page_size = HW_RXBD_RING_SIZE;
2478                 ring->pg_arr = (void **)txr->tx_desc_ring;
2479                 ring->dma_arr = txr->tx_desc_mapping;
2480                 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
2481                 ring->vmem = (void **)&txr->tx_buf_ring;
2482         }
2483 }
2484
2485 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
2486 {
2487         int i;
2488         u32 prod;
2489         struct rx_bd **rx_buf_ring;
2490
2491         rx_buf_ring = (struct rx_bd **)ring->pg_arr;
2492         for (i = 0, prod = 0; i < ring->nr_pages; i++) {
2493                 int j;
2494                 struct rx_bd *rxbd;
2495
2496                 rxbd = rx_buf_ring[i];
2497                 if (!rxbd)
2498                         continue;
2499
2500                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
2501                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
2502                         rxbd->rx_bd_opaque = prod;
2503                 }
2504         }
2505 }
2506
2507 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
2508 {
2509         struct net_device *dev = bp->dev;
2510         struct bnxt_rx_ring_info *rxr;
2511         struct bnxt_ring_struct *ring;
2512         u32 prod, type;
2513         int i;
2514
2515         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
2516                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
2517
2518         if (NET_IP_ALIGN == 2)
2519                 type |= RX_BD_FLAGS_SOP;
2520
2521         rxr = &bp->rx_ring[ring_nr];
2522         ring = &rxr->rx_ring_struct;
2523         bnxt_init_rxbd_pages(ring, type);
2524
2525         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
2526                 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1);
2527                 if (IS_ERR(rxr->xdp_prog)) {
2528                         int rc = PTR_ERR(rxr->xdp_prog);
2529
2530                         rxr->xdp_prog = NULL;
2531                         return rc;
2532                 }
2533         }
2534         prod = rxr->rx_prod;
2535         for (i = 0; i < bp->rx_ring_size; i++) {
2536                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
2537                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
2538                                     ring_nr, i, bp->rx_ring_size);
2539                         break;
2540                 }
2541                 prod = NEXT_RX(prod);
2542         }
2543         rxr->rx_prod = prod;
2544         ring->fw_ring_id = INVALID_HW_RING_ID;
2545
2546         ring = &rxr->rx_agg_ring_struct;
2547         ring->fw_ring_id = INVALID_HW_RING_ID;
2548
2549         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
2550                 return 0;
2551
2552         type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
2553                 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
2554
2555         bnxt_init_rxbd_pages(ring, type);
2556
2557         prod = rxr->rx_agg_prod;
2558         for (i = 0; i < bp->rx_agg_ring_size; i++) {
2559                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
2560                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
2561                                     ring_nr, i, bp->rx_ring_size);
2562                         break;
2563                 }
2564                 prod = NEXT_RX_AGG(prod);
2565         }
2566         rxr->rx_agg_prod = prod;
2567
2568         if (bp->flags & BNXT_FLAG_TPA) {
2569                 if (rxr->rx_tpa) {
2570                         u8 *data;
2571                         dma_addr_t mapping;
2572
2573                         for (i = 0; i < MAX_TPA; i++) {
2574                                 data = __bnxt_alloc_rx_data(bp, &mapping,
2575                                                             GFP_KERNEL);
2576                                 if (!data)
2577                                         return -ENOMEM;
2578
2579                                 rxr->rx_tpa[i].data = data;
2580                                 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
2581                                 rxr->rx_tpa[i].mapping = mapping;
2582                         }
2583                 } else {
2584                         netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
2585                         return -ENOMEM;
2586                 }
2587         }
2588
2589         return 0;
2590 }
2591
2592 static void bnxt_init_cp_rings(struct bnxt *bp)
2593 {
2594         int i;
2595
2596         for (i = 0; i < bp->cp_nr_rings; i++) {
2597                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
2598                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
2599
2600                 ring->fw_ring_id = INVALID_HW_RING_ID;
2601         }
2602 }
2603
2604 static int bnxt_init_rx_rings(struct bnxt *bp)
2605 {
2606         int i, rc = 0;
2607
2608         if (BNXT_RX_PAGE_MODE(bp)) {
2609                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
2610                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
2611         } else {
2612                 bp->rx_offset = BNXT_RX_OFFSET;
2613                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
2614         }
2615
2616         for (i = 0; i < bp->rx_nr_rings; i++) {
2617                 rc = bnxt_init_one_rx_ring(bp, i);
2618                 if (rc)
2619                         break;
2620         }
2621
2622         return rc;
2623 }
2624
2625 static int bnxt_init_tx_rings(struct bnxt *bp)
2626 {
2627         u16 i;
2628
2629         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
2630                                    MAX_SKB_FRAGS + 1);
2631
2632         for (i = 0; i < bp->tx_nr_rings; i++) {
2633                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2634                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
2635
2636                 ring->fw_ring_id = INVALID_HW_RING_ID;
2637         }
2638
2639         return 0;
2640 }
2641
2642 static void bnxt_free_ring_grps(struct bnxt *bp)
2643 {
2644         kfree(bp->grp_info);
2645         bp->grp_info = NULL;
2646 }
2647
2648 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
2649 {
2650         int i;
2651
2652         if (irq_re_init) {
2653                 bp->grp_info = kcalloc(bp->cp_nr_rings,
2654                                        sizeof(struct bnxt_ring_grp_info),
2655                                        GFP_KERNEL);
2656                 if (!bp->grp_info)
2657                         return -ENOMEM;
2658         }
2659         for (i = 0; i < bp->cp_nr_rings; i++) {
2660                 if (irq_re_init)
2661                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
2662                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
2663                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
2664                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
2665                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
2666         }
2667         return 0;
2668 }
2669
2670 static void bnxt_free_vnics(struct bnxt *bp)
2671 {
2672         kfree(bp->vnic_info);
2673         bp->vnic_info = NULL;
2674         bp->nr_vnics = 0;
2675 }
2676
2677 static int bnxt_alloc_vnics(struct bnxt *bp)
2678 {
2679         int num_vnics = 1;
2680
2681 #ifdef CONFIG_RFS_ACCEL
2682         if (bp->flags & BNXT_FLAG_RFS)
2683                 num_vnics += bp->rx_nr_rings;
2684 #endif
2685
2686         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
2687                 num_vnics++;
2688
2689         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
2690                                 GFP_KERNEL);
2691         if (!bp->vnic_info)
2692                 return -ENOMEM;
2693
2694         bp->nr_vnics = num_vnics;
2695         return 0;
2696 }
2697
2698 static void bnxt_init_vnics(struct bnxt *bp)
2699 {
2700         int i;
2701
2702         for (i = 0; i < bp->nr_vnics; i++) {
2703                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
2704
2705                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
2706                 vnic->fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID;
2707                 vnic->fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID;
2708                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
2709
2710                 if (bp->vnic_info[i].rss_hash_key) {
2711                         if (i == 0)
2712                                 prandom_bytes(vnic->rss_hash_key,
2713                                               HW_HASH_KEY_SIZE);
2714                         else
2715                                 memcpy(vnic->rss_hash_key,
2716                                        bp->vnic_info[0].rss_hash_key,
2717                                        HW_HASH_KEY_SIZE);
2718                 }
2719         }
2720 }
2721
2722 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
2723 {
2724         int pages;
2725
2726         pages = ring_size / desc_per_pg;
2727
2728         if (!pages)
2729                 return 1;
2730
2731         pages++;
2732
2733         while (pages & (pages - 1))
2734                 pages++;
2735
2736         return pages;
2737 }
2738
2739 void bnxt_set_tpa_flags(struct bnxt *bp)
2740 {
2741         bp->flags &= ~BNXT_FLAG_TPA;
2742         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
2743                 return;
2744         if (bp->dev->features & NETIF_F_LRO)
2745                 bp->flags |= BNXT_FLAG_LRO;
2746         if (bp->dev->features & NETIF_F_GRO)
2747                 bp->flags |= BNXT_FLAG_GRO;
2748 }
2749
2750 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
2751  * be set on entry.
2752  */
2753 void bnxt_set_ring_params(struct bnxt *bp)
2754 {
2755         u32 ring_size, rx_size, rx_space;
2756         u32 agg_factor = 0, agg_ring_size = 0;
2757
2758         /* 8 for CRC and VLAN */
2759         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
2760
2761         rx_space = rx_size + NET_SKB_PAD +
2762                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2763
2764         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
2765         ring_size = bp->rx_ring_size;
2766         bp->rx_agg_ring_size = 0;
2767         bp->rx_agg_nr_pages = 0;
2768
2769         if (bp->flags & BNXT_FLAG_TPA)
2770                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
2771
2772         bp->flags &= ~BNXT_FLAG_JUMBO;
2773         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
2774                 u32 jumbo_factor;
2775
2776                 bp->flags |= BNXT_FLAG_JUMBO;
2777                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
2778                 if (jumbo_factor > agg_factor)
2779                         agg_factor = jumbo_factor;
2780         }
2781         agg_ring_size = ring_size * agg_factor;
2782
2783         if (agg_ring_size) {
2784                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
2785                                                         RX_DESC_CNT);
2786                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
2787                         u32 tmp = agg_ring_size;
2788
2789                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
2790                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
2791                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
2792                                     tmp, agg_ring_size);
2793                 }
2794                 bp->rx_agg_ring_size = agg_ring_size;
2795                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
2796                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
2797                 rx_space = rx_size + NET_SKB_PAD +
2798                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2799         }
2800
2801         bp->rx_buf_use_size = rx_size;
2802         bp->rx_buf_size = rx_space;
2803
2804         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
2805         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
2806
2807         ring_size = bp->tx_ring_size;
2808         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
2809         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
2810
2811         ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
2812         bp->cp_ring_size = ring_size;
2813
2814         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
2815         if (bp->cp_nr_pages > MAX_CP_PAGES) {
2816                 bp->cp_nr_pages = MAX_CP_PAGES;
2817                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
2818                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
2819                             ring_size, bp->cp_ring_size);
2820         }
2821         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
2822         bp->cp_ring_mask = bp->cp_bit - 1;
2823 }
2824
2825 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
2826 {
2827         if (page_mode) {
2828                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
2829                         return -EOPNOTSUPP;
2830                 bp->dev->max_mtu = BNXT_MAX_PAGE_MODE_MTU;
2831                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
2832                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
2833                 bp->dev->hw_features &= ~NETIF_F_LRO;
2834                 bp->dev->features &= ~NETIF_F_LRO;
2835                 bp->rx_dir = DMA_BIDIRECTIONAL;
2836                 bp->rx_skb_func = bnxt_rx_page_skb;
2837         } else {
2838                 bp->dev->max_mtu = BNXT_MAX_MTU;
2839                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
2840                 bp->rx_dir = DMA_FROM_DEVICE;
2841                 bp->rx_skb_func = bnxt_rx_skb;
2842         }
2843         return 0;
2844 }
2845
2846 static void bnxt_free_vnic_attributes(struct bnxt *bp)
2847 {
2848         int i;
2849         struct bnxt_vnic_info *vnic;
2850         struct pci_dev *pdev = bp->pdev;
2851
2852         if (!bp->vnic_info)
2853                 return;
2854
2855         for (i = 0; i < bp->nr_vnics; i++) {
2856                 vnic = &bp->vnic_info[i];
2857
2858                 kfree(vnic->fw_grp_ids);
2859                 vnic->fw_grp_ids = NULL;
2860
2861                 kfree(vnic->uc_list);
2862                 vnic->uc_list = NULL;
2863
2864                 if (vnic->mc_list) {
2865                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
2866                                           vnic->mc_list, vnic->mc_list_mapping);
2867                         vnic->mc_list = NULL;
2868                 }
2869
2870                 if (vnic->rss_table) {
2871                         dma_free_coherent(&pdev->dev, PAGE_SIZE,
2872                                           vnic->rss_table,
2873                                           vnic->rss_table_dma_addr);
2874                         vnic->rss_table = NULL;
2875                 }
2876
2877                 vnic->rss_hash_key = NULL;
2878                 vnic->flags = 0;
2879         }
2880 }
2881
2882 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
2883 {
2884         int i, rc = 0, size;
2885         struct bnxt_vnic_info *vnic;
2886         struct pci_dev *pdev = bp->pdev;
2887         int max_rings;
2888
2889         for (i = 0; i < bp->nr_vnics; i++) {
2890                 vnic = &bp->vnic_info[i];
2891
2892                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
2893                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
2894
2895                         if (mem_size > 0) {
2896                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
2897                                 if (!vnic->uc_list) {
2898                                         rc = -ENOMEM;
2899                                         goto out;
2900                                 }
2901                         }
2902                 }
2903
2904                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
2905                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
2906                         vnic->mc_list =
2907                                 dma_alloc_coherent(&pdev->dev,
2908                                                    vnic->mc_list_size,
2909                                                    &vnic->mc_list_mapping,
2910                                                    GFP_KERNEL);
2911                         if (!vnic->mc_list) {
2912                                 rc = -ENOMEM;
2913                                 goto out;
2914                         }
2915                 }
2916
2917                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
2918                         max_rings = bp->rx_nr_rings;
2919                 else
2920                         max_rings = 1;
2921
2922                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
2923                 if (!vnic->fw_grp_ids) {
2924                         rc = -ENOMEM;
2925                         goto out;
2926                 }
2927
2928                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
2929                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
2930                         continue;
2931
2932                 /* Allocate rss table and hash key */
2933                 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2934                                                      &vnic->rss_table_dma_addr,
2935                                                      GFP_KERNEL);
2936                 if (!vnic->rss_table) {
2937                         rc = -ENOMEM;
2938                         goto out;
2939                 }
2940
2941                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
2942
2943                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
2944                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
2945         }
2946         return 0;
2947
2948 out:
2949         return rc;
2950 }
2951
2952 static void bnxt_free_hwrm_resources(struct bnxt *bp)
2953 {
2954         struct pci_dev *pdev = bp->pdev;
2955
2956         dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
2957                           bp->hwrm_cmd_resp_dma_addr);
2958
2959         bp->hwrm_cmd_resp_addr = NULL;
2960         if (bp->hwrm_dbg_resp_addr) {
2961                 dma_free_coherent(&pdev->dev, HWRM_DBG_REG_BUF_SIZE,
2962                                   bp->hwrm_dbg_resp_addr,
2963                                   bp->hwrm_dbg_resp_dma_addr);
2964
2965                 bp->hwrm_dbg_resp_addr = NULL;
2966         }
2967 }
2968
2969 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
2970 {
2971         struct pci_dev *pdev = bp->pdev;
2972
2973         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2974                                                    &bp->hwrm_cmd_resp_dma_addr,
2975                                                    GFP_KERNEL);
2976         if (!bp->hwrm_cmd_resp_addr)
2977                 return -ENOMEM;
2978         bp->hwrm_dbg_resp_addr = dma_alloc_coherent(&pdev->dev,
2979                                                     HWRM_DBG_REG_BUF_SIZE,
2980                                                     &bp->hwrm_dbg_resp_dma_addr,
2981                                                     GFP_KERNEL);
2982         if (!bp->hwrm_dbg_resp_addr)
2983                 netdev_warn(bp->dev, "fail to alloc debug register dma mem\n");
2984
2985         return 0;
2986 }
2987
2988 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
2989 {
2990         if (bp->hwrm_short_cmd_req_addr) {
2991                 struct pci_dev *pdev = bp->pdev;
2992
2993                 dma_free_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN,
2994                                   bp->hwrm_short_cmd_req_addr,
2995                                   bp->hwrm_short_cmd_req_dma_addr);
2996                 bp->hwrm_short_cmd_req_addr = NULL;
2997         }
2998 }
2999
3000 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3001 {
3002         struct pci_dev *pdev = bp->pdev;
3003
3004         bp->hwrm_short_cmd_req_addr =
3005                 dma_alloc_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN,
3006                                    &bp->hwrm_short_cmd_req_dma_addr,
3007                                    GFP_KERNEL);
3008         if (!bp->hwrm_short_cmd_req_addr)
3009                 return -ENOMEM;
3010
3011         return 0;
3012 }
3013
3014 static void bnxt_free_stats(struct bnxt *bp)
3015 {
3016         u32 size, i;
3017         struct pci_dev *pdev = bp->pdev;
3018
3019         if (bp->hw_rx_port_stats) {
3020                 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
3021                                   bp->hw_rx_port_stats,
3022                                   bp->hw_rx_port_stats_map);
3023                 bp->hw_rx_port_stats = NULL;
3024                 bp->flags &= ~BNXT_FLAG_PORT_STATS;
3025         }
3026
3027         if (!bp->bnapi)
3028                 return;
3029
3030         size = sizeof(struct ctx_hw_stats);
3031
3032         for (i = 0; i < bp->cp_nr_rings; i++) {
3033                 struct bnxt_napi *bnapi = bp->bnapi[i];
3034                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3035
3036                 if (cpr->hw_stats) {
3037                         dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
3038                                           cpr->hw_stats_map);
3039                         cpr->hw_stats = NULL;
3040                 }
3041         }
3042 }
3043
3044 static int bnxt_alloc_stats(struct bnxt *bp)
3045 {
3046         u32 size, i;
3047         struct pci_dev *pdev = bp->pdev;
3048
3049         size = sizeof(struct ctx_hw_stats);
3050
3051         for (i = 0; i < bp->cp_nr_rings; i++) {
3052                 struct bnxt_napi *bnapi = bp->bnapi[i];
3053                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3054
3055                 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
3056                                                    &cpr->hw_stats_map,
3057                                                    GFP_KERNEL);
3058                 if (!cpr->hw_stats)
3059                         return -ENOMEM;
3060
3061                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3062         }
3063
3064         if (BNXT_PF(bp) && bp->chip_num != CHIP_NUM_58700) {
3065                 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
3066                                          sizeof(struct tx_port_stats) + 1024;
3067
3068                 bp->hw_rx_port_stats =
3069                         dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
3070                                            &bp->hw_rx_port_stats_map,
3071                                            GFP_KERNEL);
3072                 if (!bp->hw_rx_port_stats)
3073                         return -ENOMEM;
3074
3075                 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) +
3076                                        512;
3077                 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
3078                                            sizeof(struct rx_port_stats) + 512;
3079                 bp->flags |= BNXT_FLAG_PORT_STATS;
3080         }
3081         return 0;
3082 }
3083
3084 static void bnxt_clear_ring_indices(struct bnxt *bp)
3085 {
3086         int i;
3087
3088         if (!bp->bnapi)
3089                 return;
3090
3091         for (i = 0; i < bp->cp_nr_rings; i++) {
3092                 struct bnxt_napi *bnapi = bp->bnapi[i];
3093                 struct bnxt_cp_ring_info *cpr;
3094                 struct bnxt_rx_ring_info *rxr;
3095                 struct bnxt_tx_ring_info *txr;
3096
3097                 if (!bnapi)
3098                         continue;
3099
3100                 cpr = &bnapi->cp_ring;
3101                 cpr->cp_raw_cons = 0;
3102
3103                 txr = bnapi->tx_ring;
3104                 if (txr) {
3105                         txr->tx_prod = 0;
3106                         txr->tx_cons = 0;
3107                 }
3108
3109                 rxr = bnapi->rx_ring;
3110                 if (rxr) {
3111                         rxr->rx_prod = 0;
3112                         rxr->rx_agg_prod = 0;
3113                         rxr->rx_sw_agg_prod = 0;
3114                         rxr->rx_next_cons = 0;
3115                 }
3116         }
3117 }
3118
3119 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3120 {
3121 #ifdef CONFIG_RFS_ACCEL
3122         int i;
3123
3124         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
3125          * safe to delete the hash table.
3126          */
3127         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
3128                 struct hlist_head *head;
3129                 struct hlist_node *tmp;
3130                 struct bnxt_ntuple_filter *fltr;
3131
3132                 head = &bp->ntp_fltr_hash_tbl[i];
3133                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
3134                         hlist_del(&fltr->hash);
3135                         kfree(fltr);
3136                 }
3137         }
3138         if (irq_reinit) {
3139                 kfree(bp->ntp_fltr_bmap);
3140                 bp->ntp_fltr_bmap = NULL;
3141         }
3142         bp->ntp_fltr_count = 0;
3143 #endif
3144 }
3145
3146 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
3147 {
3148 #ifdef CONFIG_RFS_ACCEL
3149         int i, rc = 0;
3150
3151         if (!(bp->flags & BNXT_FLAG_RFS))
3152                 return 0;
3153
3154         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
3155                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
3156
3157         bp->ntp_fltr_count = 0;
3158         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
3159                                     sizeof(long),
3160                                     GFP_KERNEL);
3161
3162         if (!bp->ntp_fltr_bmap)
3163                 rc = -ENOMEM;
3164
3165         return rc;
3166 #else
3167         return 0;
3168 #endif
3169 }
3170
3171 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
3172 {
3173         bnxt_free_vnic_attributes(bp);
3174         bnxt_free_tx_rings(bp);
3175         bnxt_free_rx_rings(bp);
3176         bnxt_free_cp_rings(bp);
3177         bnxt_free_ntp_fltrs(bp, irq_re_init);
3178         if (irq_re_init) {
3179                 bnxt_free_stats(bp);
3180                 bnxt_free_ring_grps(bp);
3181                 bnxt_free_vnics(bp);
3182                 kfree(bp->tx_ring_map);
3183                 bp->tx_ring_map = NULL;
3184                 kfree(bp->tx_ring);
3185                 bp->tx_ring = NULL;
3186                 kfree(bp->rx_ring);
3187                 bp->rx_ring = NULL;
3188                 kfree(bp->bnapi);
3189                 bp->bnapi = NULL;
3190         } else {
3191                 bnxt_clear_ring_indices(bp);
3192         }
3193 }
3194
3195 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
3196 {
3197         int i, j, rc, size, arr_size;
3198         void *bnapi;
3199
3200         if (irq_re_init) {
3201                 /* Allocate bnapi mem pointer array and mem block for
3202                  * all queues
3203                  */
3204                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
3205                                 bp->cp_nr_rings);
3206                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
3207                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
3208                 if (!bnapi)
3209                         return -ENOMEM;
3210
3211                 bp->bnapi = bnapi;
3212                 bnapi += arr_size;
3213                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
3214                         bp->bnapi[i] = bnapi;
3215                         bp->bnapi[i]->index = i;
3216                         bp->bnapi[i]->bp = bp;
3217                 }
3218
3219                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
3220                                       sizeof(struct bnxt_rx_ring_info),
3221                                       GFP_KERNEL);
3222                 if (!bp->rx_ring)
3223                         return -ENOMEM;
3224
3225                 for (i = 0; i < bp->rx_nr_rings; i++) {
3226                         bp->rx_ring[i].bnapi = bp->bnapi[i];
3227                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
3228                 }
3229
3230                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
3231                                       sizeof(struct bnxt_tx_ring_info),
3232                                       GFP_KERNEL);
3233                 if (!bp->tx_ring)
3234                         return -ENOMEM;
3235
3236                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
3237                                           GFP_KERNEL);
3238
3239                 if (!bp->tx_ring_map)
3240                         return -ENOMEM;
3241
3242                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
3243                         j = 0;
3244                 else
3245                         j = bp->rx_nr_rings;
3246
3247                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
3248                         bp->tx_ring[i].bnapi = bp->bnapi[j];
3249                         bp->bnapi[j]->tx_ring = &bp->tx_ring[i];
3250                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
3251                         if (i >= bp->tx_nr_rings_xdp) {
3252                                 bp->tx_ring[i].txq_index = i -
3253                                         bp->tx_nr_rings_xdp;
3254                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
3255                         } else {
3256                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
3257                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
3258                         }
3259                 }
3260
3261                 rc = bnxt_alloc_stats(bp);
3262                 if (rc)
3263                         goto alloc_mem_err;
3264
3265                 rc = bnxt_alloc_ntp_fltrs(bp);
3266                 if (rc)
3267                         goto alloc_mem_err;
3268
3269                 rc = bnxt_alloc_vnics(bp);
3270                 if (rc)
3271                         goto alloc_mem_err;
3272         }
3273
3274         bnxt_init_ring_struct(bp);
3275
3276         rc = bnxt_alloc_rx_rings(bp);
3277         if (rc)
3278                 goto alloc_mem_err;
3279
3280         rc = bnxt_alloc_tx_rings(bp);
3281         if (rc)
3282                 goto alloc_mem_err;
3283
3284         rc = bnxt_alloc_cp_rings(bp);
3285         if (rc)
3286                 goto alloc_mem_err;
3287
3288         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
3289                                   BNXT_VNIC_UCAST_FLAG;
3290         rc = bnxt_alloc_vnic_attributes(bp);
3291         if (rc)
3292                 goto alloc_mem_err;
3293         return 0;
3294
3295 alloc_mem_err:
3296         bnxt_free_mem(bp, true);
3297         return rc;
3298 }
3299
3300 static void bnxt_disable_int(struct bnxt *bp)
3301 {
3302         int i;
3303
3304         if (!bp->bnapi)
3305                 return;
3306
3307         for (i = 0; i < bp->cp_nr_rings; i++) {
3308                 struct bnxt_napi *bnapi = bp->bnapi[i];
3309                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3310                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3311
3312                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
3313                         BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
3314         }
3315 }
3316
3317 static void bnxt_disable_int_sync(struct bnxt *bp)
3318 {
3319         int i;
3320
3321         atomic_inc(&bp->intr_sem);
3322
3323         bnxt_disable_int(bp);
3324         for (i = 0; i < bp->cp_nr_rings; i++)
3325                 synchronize_irq(bp->irq_tbl[i].vector);
3326 }
3327
3328 static void bnxt_enable_int(struct bnxt *bp)
3329 {
3330         int i;
3331
3332         atomic_set(&bp->intr_sem, 0);
3333         for (i = 0; i < bp->cp_nr_rings; i++) {
3334                 struct bnxt_napi *bnapi = bp->bnapi[i];
3335                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3336
3337                 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
3338         }
3339 }
3340
3341 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
3342                             u16 cmpl_ring, u16 target_id)
3343 {
3344         struct input *req = request;
3345
3346         req->req_type = cpu_to_le16(req_type);
3347         req->cmpl_ring = cpu_to_le16(cmpl_ring);
3348         req->target_id = cpu_to_le16(target_id);
3349         req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
3350 }
3351
3352 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
3353                                  int timeout, bool silent)
3354 {
3355         int i, intr_process, rc, tmo_count;
3356         struct input *req = msg;
3357         u32 *data = msg;
3358         __le32 *resp_len, *valid;
3359         u16 cp_ring_id, len = 0;
3360         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
3361         u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
3362
3363         req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++);
3364         memset(resp, 0, PAGE_SIZE);
3365         cp_ring_id = le16_to_cpu(req->cmpl_ring);
3366         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
3367
3368         if (bp->flags & BNXT_FLAG_SHORT_CMD) {
3369                 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
3370                 struct hwrm_short_input short_input = {0};
3371
3372                 memcpy(short_cmd_req, req, msg_len);
3373                 memset(short_cmd_req + msg_len, 0, BNXT_HWRM_MAX_REQ_LEN -
3374                                                    msg_len);
3375
3376                 short_input.req_type = req->req_type;
3377                 short_input.signature =
3378                                 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
3379                 short_input.size = cpu_to_le16(msg_len);
3380                 short_input.req_addr =
3381                         cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
3382
3383                 data = (u32 *)&short_input;
3384                 msg_len = sizeof(short_input);
3385
3386                 /* Sync memory write before updating doorbell */
3387                 wmb();
3388
3389                 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
3390         }
3391
3392         /* Write request msg to hwrm channel */
3393         __iowrite32_copy(bp->bar0, data, msg_len / 4);
3394
3395         for (i = msg_len; i < max_req_len; i += 4)
3396                 writel(0, bp->bar0 + i);
3397
3398         /* currently supports only one outstanding message */
3399         if (intr_process)
3400                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
3401
3402         /* Ring channel doorbell */
3403         writel(1, bp->bar0 + 0x100);
3404
3405         if (!timeout)
3406                 timeout = DFLT_HWRM_CMD_TIMEOUT;
3407
3408         i = 0;
3409         tmo_count = timeout * 40;
3410         if (intr_process) {
3411                 /* Wait until hwrm response cmpl interrupt is processed */
3412                 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID &&
3413                        i++ < tmo_count) {
3414                         usleep_range(25, 40);
3415                 }
3416
3417                 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) {
3418                         netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
3419                                    le16_to_cpu(req->req_type));
3420                         return -1;
3421                 }
3422         } else {
3423                 /* Check if response len is updated */
3424                 resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET;
3425                 for (i = 0; i < tmo_count; i++) {
3426                         len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
3427                               HWRM_RESP_LEN_SFT;
3428                         if (len)
3429                                 break;
3430                         usleep_range(25, 40);
3431                 }
3432
3433                 if (i >= tmo_count) {
3434                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
3435                                    timeout, le16_to_cpu(req->req_type),
3436                                    le16_to_cpu(req->seq_id), len);
3437                         return -1;
3438                 }
3439
3440                 /* Last word of resp contains valid bit */
3441                 valid = bp->hwrm_cmd_resp_addr + len - 4;
3442                 for (i = 0; i < 5; i++) {
3443                         if (le32_to_cpu(*valid) & HWRM_RESP_VALID_MASK)
3444                                 break;
3445                         udelay(1);
3446                 }
3447
3448                 if (i >= 5) {
3449                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
3450                                    timeout, le16_to_cpu(req->req_type),
3451                                    le16_to_cpu(req->seq_id), len, *valid);
3452                         return -1;
3453                 }
3454         }
3455
3456         rc = le16_to_cpu(resp->error_code);
3457         if (rc && !silent)
3458                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
3459                            le16_to_cpu(resp->req_type),
3460                            le16_to_cpu(resp->seq_id), rc);
3461         return rc;
3462 }
3463
3464 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
3465 {
3466         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
3467 }
3468
3469 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
3470                               int timeout)
3471 {
3472         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
3473 }
3474
3475 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
3476 {
3477         int rc;
3478
3479         mutex_lock(&bp->hwrm_cmd_lock);
3480         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
3481         mutex_unlock(&bp->hwrm_cmd_lock);
3482         return rc;
3483 }
3484
3485 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
3486                              int timeout)
3487 {
3488         int rc;
3489
3490         mutex_lock(&bp->hwrm_cmd_lock);
3491         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
3492         mutex_unlock(&bp->hwrm_cmd_lock);
3493         return rc;
3494 }
3495
3496 int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap,
3497                                      int bmap_size)
3498 {
3499         struct hwrm_func_drv_rgtr_input req = {0};
3500         DECLARE_BITMAP(async_events_bmap, 256);
3501         u32 *events = (u32 *)async_events_bmap;
3502         int i;
3503
3504         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
3505
3506         req.enables =
3507                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
3508
3509         memset(async_events_bmap, 0, sizeof(async_events_bmap));
3510         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
3511                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
3512
3513         if (bmap && bmap_size) {
3514                 for (i = 0; i < bmap_size; i++) {
3515                         if (test_bit(i, bmap))
3516                                 __set_bit(i, async_events_bmap);
3517                 }
3518         }
3519
3520         for (i = 0; i < 8; i++)
3521                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
3522
3523         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3524 }
3525
3526 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
3527 {
3528         struct hwrm_func_drv_rgtr_input req = {0};
3529
3530         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
3531
3532         req.enables =
3533                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
3534                             FUNC_DRV_RGTR_REQ_ENABLES_VER);
3535
3536         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
3537         req.ver_maj = DRV_VER_MAJ;
3538         req.ver_min = DRV_VER_MIN;
3539         req.ver_upd = DRV_VER_UPD;
3540
3541         if (BNXT_PF(bp)) {
3542                 u32 data[8];
3543                 int i;
3544
3545                 memset(data, 0, sizeof(data));
3546                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
3547                         u16 cmd = bnxt_vf_req_snif[i];
3548                         unsigned int bit, idx;
3549
3550                         idx = cmd / 32;
3551                         bit = cmd % 32;
3552                         data[idx] |= 1 << bit;
3553                 }
3554
3555                 for (i = 0; i < 8; i++)
3556                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
3557
3558                 req.enables |=
3559                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
3560         }
3561
3562         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3563 }
3564
3565 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
3566 {
3567         struct hwrm_func_drv_unrgtr_input req = {0};
3568
3569         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
3570         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3571 }
3572
3573 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
3574 {
3575         u32 rc = 0;
3576         struct hwrm_tunnel_dst_port_free_input req = {0};
3577
3578         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
3579         req.tunnel_type = tunnel_type;
3580
3581         switch (tunnel_type) {
3582         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
3583                 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
3584                 break;
3585         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
3586                 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
3587                 break;
3588         default:
3589                 break;
3590         }
3591
3592         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3593         if (rc)
3594                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
3595                            rc);
3596         return rc;
3597 }
3598
3599 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
3600                                            u8 tunnel_type)
3601 {
3602         u32 rc = 0;
3603         struct hwrm_tunnel_dst_port_alloc_input req = {0};
3604         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3605
3606         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
3607
3608         req.tunnel_type = tunnel_type;
3609         req.tunnel_dst_port_val = port;
3610
3611         mutex_lock(&bp->hwrm_cmd_lock);
3612         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3613         if (rc) {
3614                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
3615                            rc);
3616                 goto err_out;
3617         }
3618
3619         switch (tunnel_type) {
3620         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
3621                 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
3622                 break;
3623         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
3624                 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
3625                 break;
3626         default:
3627                 break;
3628         }
3629
3630 err_out:
3631         mutex_unlock(&bp->hwrm_cmd_lock);
3632         return rc;
3633 }
3634
3635 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
3636 {
3637         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
3638         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3639
3640         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
3641         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
3642
3643         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
3644         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
3645         req.mask = cpu_to_le32(vnic->rx_mask);
3646         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3647 }
3648
3649 #ifdef CONFIG_RFS_ACCEL
3650 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
3651                                             struct bnxt_ntuple_filter *fltr)
3652 {
3653         struct hwrm_cfa_ntuple_filter_free_input req = {0};
3654
3655         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
3656         req.ntuple_filter_id = fltr->filter_id;
3657         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3658 }
3659
3660 #define BNXT_NTP_FLTR_FLAGS                                     \
3661         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
3662          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
3663          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
3664          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
3665          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
3666          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
3667          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
3668          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
3669          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
3670          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
3671          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
3672          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
3673          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
3674          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
3675
3676 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
3677                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
3678
3679 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
3680                                              struct bnxt_ntuple_filter *fltr)
3681 {
3682         int rc = 0;
3683         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
3684         struct hwrm_cfa_ntuple_filter_alloc_output *resp =
3685                 bp->hwrm_cmd_resp_addr;
3686         struct flow_keys *keys = &fltr->fkeys;
3687         struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1];
3688
3689         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
3690         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
3691
3692         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
3693
3694         req.ethertype = htons(ETH_P_IP);
3695         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
3696         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
3697         req.ip_protocol = keys->basic.ip_proto;
3698
3699         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
3700                 int i;
3701
3702                 req.ethertype = htons(ETH_P_IPV6);
3703                 req.ip_addr_type =
3704                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
3705                 *(struct in6_addr *)&req.src_ipaddr[0] =
3706                         keys->addrs.v6addrs.src;
3707                 *(struct in6_addr *)&req.dst_ipaddr[0] =
3708                         keys->addrs.v6addrs.dst;
3709                 for (i = 0; i < 4; i++) {
3710                         req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
3711                         req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
3712                 }
3713         } else {
3714                 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
3715                 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3716                 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
3717                 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3718         }
3719         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
3720                 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
3721                 req.tunnel_type =
3722                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
3723         }
3724
3725         req.src_port = keys->ports.src;
3726         req.src_port_mask = cpu_to_be16(0xffff);
3727         req.dst_port = keys->ports.dst;
3728         req.dst_port_mask = cpu_to_be16(0xffff);
3729
3730         req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
3731         mutex_lock(&bp->hwrm_cmd_lock);
3732         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3733         if (!rc)
3734                 fltr->filter_id = resp->ntuple_filter_id;
3735         mutex_unlock(&bp->hwrm_cmd_lock);
3736         return rc;
3737 }
3738 #endif
3739
3740 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
3741                                      u8 *mac_addr)
3742 {
3743         u32 rc = 0;
3744         struct hwrm_cfa_l2_filter_alloc_input req = {0};
3745         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3746
3747         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
3748         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
3749         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
3750                 req.flags |=
3751                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
3752         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
3753         req.enables =
3754                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
3755                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
3756                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
3757         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
3758         req.l2_addr_mask[0] = 0xff;
3759         req.l2_addr_mask[1] = 0xff;
3760         req.l2_addr_mask[2] = 0xff;
3761         req.l2_addr_mask[3] = 0xff;
3762         req.l2_addr_mask[4] = 0xff;
3763         req.l2_addr_mask[5] = 0xff;
3764
3765         mutex_lock(&bp->hwrm_cmd_lock);
3766         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3767         if (!rc)
3768                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
3769                                                         resp->l2_filter_id;
3770         mutex_unlock(&bp->hwrm_cmd_lock);
3771         return rc;
3772 }
3773
3774 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
3775 {
3776         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
3777         int rc = 0;
3778
3779         /* Any associated ntuple filters will also be cleared by firmware. */
3780         mutex_lock(&bp->hwrm_cmd_lock);
3781         for (i = 0; i < num_of_vnics; i++) {
3782                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3783
3784                 for (j = 0; j < vnic->uc_filter_count; j++) {
3785                         struct hwrm_cfa_l2_filter_free_input req = {0};
3786
3787                         bnxt_hwrm_cmd_hdr_init(bp, &req,
3788                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
3789
3790                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
3791
3792                         rc = _hwrm_send_message(bp, &req, sizeof(req),
3793                                                 HWRM_CMD_TIMEOUT);
3794                 }
3795                 vnic->uc_filter_count = 0;
3796         }
3797         mutex_unlock(&bp->hwrm_cmd_lock);
3798
3799         return rc;
3800 }
3801
3802 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
3803 {
3804         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3805         struct hwrm_vnic_tpa_cfg_input req = {0};
3806
3807         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
3808
3809         if (tpa_flags) {
3810                 u16 mss = bp->dev->mtu - 40;
3811                 u32 nsegs, n, segs = 0, flags;
3812
3813                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
3814                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
3815                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
3816                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
3817                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
3818                 if (tpa_flags & BNXT_FLAG_GRO)
3819                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
3820
3821                 req.flags = cpu_to_le32(flags);
3822
3823                 req.enables =
3824                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
3825                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
3826                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
3827
3828                 /* Number of segs are log2 units, and first packet is not
3829                  * included as part of this units.
3830                  */
3831                 if (mss <= BNXT_RX_PAGE_SIZE) {
3832                         n = BNXT_RX_PAGE_SIZE / mss;
3833                         nsegs = (MAX_SKB_FRAGS - 1) * n;
3834                 } else {
3835                         n = mss / BNXT_RX_PAGE_SIZE;
3836                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
3837                                 n++;
3838                         nsegs = (MAX_SKB_FRAGS - n) / n;
3839                 }
3840
3841                 segs = ilog2(nsegs);
3842                 req.max_agg_segs = cpu_to_le16(segs);
3843                 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX);
3844
3845                 req.min_agg_len = cpu_to_le32(512);
3846         }
3847         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3848
3849         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3850 }
3851
3852 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
3853 {
3854         u32 i, j, max_rings;
3855         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3856         struct hwrm_vnic_rss_cfg_input req = {0};
3857
3858         if (vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
3859                 return 0;
3860
3861         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
3862         if (set_rss) {
3863                 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
3864                 if (vnic->flags & BNXT_VNIC_RSS_FLAG) {
3865                         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3866                                 max_rings = bp->rx_nr_rings - 1;
3867                         else
3868                                 max_rings = bp->rx_nr_rings;
3869                 } else {
3870                         max_rings = 1;
3871                 }
3872
3873                 /* Fill the RSS indirection table with ring group ids */
3874                 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
3875                         if (j == max_rings)
3876                                 j = 0;
3877                         vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
3878                 }
3879
3880                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
3881                 req.hash_key_tbl_addr =
3882                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
3883         }
3884         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
3885         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3886 }
3887
3888 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
3889 {
3890         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3891         struct hwrm_vnic_plcmodes_cfg_input req = {0};
3892
3893         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
3894         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
3895                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
3896                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
3897         req.enables =
3898                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
3899                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
3900         /* thresholds not implemented in firmware yet */
3901         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
3902         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
3903         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
3904         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3905 }
3906
3907 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
3908                                         u16 ctx_idx)
3909 {
3910         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
3911
3912         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
3913         req.rss_cos_lb_ctx_id =
3914                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
3915
3916         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3917         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
3918 }
3919
3920 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
3921 {
3922         int i, j;
3923
3924         for (i = 0; i < bp->nr_vnics; i++) {
3925                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3926
3927                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
3928                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
3929                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
3930                 }
3931         }
3932         bp->rsscos_nr_ctxs = 0;
3933 }
3934
3935 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
3936 {
3937         int rc;
3938         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
3939         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
3940                                                 bp->hwrm_cmd_resp_addr;
3941
3942         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
3943                                -1);
3944
3945         mutex_lock(&bp->hwrm_cmd_lock);
3946         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3947         if (!rc)
3948                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
3949                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
3950         mutex_unlock(&bp->hwrm_cmd_lock);
3951
3952         return rc;
3953 }
3954
3955 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
3956 {
3957         unsigned int ring = 0, grp_idx;
3958         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3959         struct hwrm_vnic_cfg_input req = {0};
3960         u16 def_vlan = 0;
3961
3962         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
3963
3964         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
3965         /* Only RSS support for now TBD: COS & LB */
3966         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
3967                 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
3968                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
3969                                            VNIC_CFG_REQ_ENABLES_MRU);
3970         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
3971                 req.rss_rule =
3972                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
3973                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
3974                                            VNIC_CFG_REQ_ENABLES_MRU);
3975                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
3976         } else {
3977                 req.rss_rule = cpu_to_le16(0xffff);
3978         }
3979
3980         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
3981             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
3982                 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
3983                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
3984         } else {
3985                 req.cos_rule = cpu_to_le16(0xffff);
3986         }
3987
3988         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3989                 ring = 0;
3990         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
3991                 ring = vnic_id - 1;
3992         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
3993                 ring = bp->rx_nr_rings - 1;
3994
3995         grp_idx = bp->rx_ring[ring].bnapi->index;
3996         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3997         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
3998
3999         req.lb_rule = cpu_to_le16(0xffff);
4000         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
4001                               VLAN_HLEN);
4002
4003 #ifdef CONFIG_BNXT_SRIOV
4004         if (BNXT_VF(bp))
4005                 def_vlan = bp->vf.vlan;
4006 #endif
4007         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
4008                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
4009         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
4010                 req.flags |=
4011                         cpu_to_le32(VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE);
4012
4013         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4014 }
4015
4016 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
4017 {
4018         u32 rc = 0;
4019
4020         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
4021                 struct hwrm_vnic_free_input req = {0};
4022
4023                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
4024                 req.vnic_id =
4025                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
4026
4027                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4028                 if (rc)
4029                         return rc;
4030                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
4031         }
4032         return rc;
4033 }
4034
4035 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
4036 {
4037         u16 i;
4038
4039         for (i = 0; i < bp->nr_vnics; i++)
4040                 bnxt_hwrm_vnic_free_one(bp, i);
4041 }
4042
4043 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
4044                                 unsigned int start_rx_ring_idx,
4045                                 unsigned int nr_rings)
4046 {
4047         int rc = 0;
4048         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
4049         struct hwrm_vnic_alloc_input req = {0};
4050         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4051
4052         /* map ring groups to this vnic */
4053         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
4054                 grp_idx = bp->rx_ring[i].bnapi->index;
4055                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
4056                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
4057                                    j, nr_rings);
4058                         break;
4059                 }
4060                 bp->vnic_info[vnic_id].fw_grp_ids[j] =
4061                                         bp->grp_info[grp_idx].fw_grp_id;
4062         }
4063
4064         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID;
4065         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID;
4066         if (vnic_id == 0)
4067                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
4068
4069         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
4070
4071         mutex_lock(&bp->hwrm_cmd_lock);
4072         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4073         if (!rc)
4074                 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id);
4075         mutex_unlock(&bp->hwrm_cmd_lock);
4076         return rc;
4077 }
4078
4079 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
4080 {
4081         struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4082         struct hwrm_vnic_qcaps_input req = {0};
4083         int rc;
4084
4085         if (bp->hwrm_spec_code < 0x10600)
4086                 return 0;
4087
4088         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
4089         mutex_lock(&bp->hwrm_cmd_lock);
4090         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4091         if (!rc) {
4092                 if (resp->flags &
4093                     cpu_to_le32(VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
4094                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
4095         }
4096         mutex_unlock(&bp->hwrm_cmd_lock);
4097         return rc;
4098 }
4099
4100 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
4101 {
4102         u16 i;
4103         u32 rc = 0;
4104
4105         mutex_lock(&bp->hwrm_cmd_lock);
4106         for (i = 0; i < bp->rx_nr_rings; i++) {
4107                 struct hwrm_ring_grp_alloc_input req = {0};
4108                 struct hwrm_ring_grp_alloc_output *resp =
4109                                         bp->hwrm_cmd_resp_addr;
4110                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
4111
4112                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
4113
4114                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
4115                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
4116                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
4117                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
4118
4119                 rc = _hwrm_send_message(bp, &req, sizeof(req),
4120                                         HWRM_CMD_TIMEOUT);
4121                 if (rc)
4122                         break;
4123
4124                 bp->grp_info[grp_idx].fw_grp_id =
4125                         le32_to_cpu(resp->ring_group_id);
4126         }
4127         mutex_unlock(&bp->hwrm_cmd_lock);
4128         return rc;
4129 }
4130
4131 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
4132 {
4133         u16 i;
4134         u32 rc = 0;
4135         struct hwrm_ring_grp_free_input req = {0};
4136
4137         if (!bp->grp_info)
4138                 return 0;
4139
4140         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
4141
4142         mutex_lock(&bp->hwrm_cmd_lock);
4143         for (i = 0; i < bp->cp_nr_rings; i++) {
4144                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
4145                         continue;
4146                 req.ring_group_id =
4147                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
4148
4149                 rc = _hwrm_send_message(bp, &req, sizeof(req),
4150                                         HWRM_CMD_TIMEOUT);
4151                 if (rc)
4152                         break;
4153                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
4154         }
4155         mutex_unlock(&bp->hwrm_cmd_lock);
4156         return rc;
4157 }
4158
4159 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
4160                                     struct bnxt_ring_struct *ring,
4161                                     u32 ring_type, u32 map_index,
4162                                     u32 stats_ctx_id)
4163 {
4164         int rc = 0, err = 0;
4165         struct hwrm_ring_alloc_input req = {0};
4166         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4167         u16 ring_id;
4168
4169         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
4170
4171         req.enables = 0;
4172         if (ring->nr_pages > 1) {
4173                 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map);
4174                 /* Page size is in log2 units */
4175                 req.page_size = BNXT_PAGE_SHIFT;
4176                 req.page_tbl_depth = 1;
4177         } else {
4178                 req.page_tbl_addr =  cpu_to_le64(ring->dma_arr[0]);
4179         }
4180         req.fbo = 0;
4181         /* Association of ring index with doorbell index and MSIX number */
4182         req.logical_id = cpu_to_le16(map_index);
4183
4184         switch (ring_type) {
4185         case HWRM_RING_ALLOC_TX:
4186                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
4187                 /* Association of transmit ring with completion ring */
4188                 req.cmpl_ring_id =
4189                         cpu_to_le16(bp->grp_info[map_index].cp_fw_ring_id);
4190                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
4191                 req.stat_ctx_id = cpu_to_le32(stats_ctx_id);
4192                 req.queue_id = cpu_to_le16(ring->queue_id);
4193                 break;
4194         case HWRM_RING_ALLOC_RX:
4195                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
4196                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
4197                 break;
4198         case HWRM_RING_ALLOC_AGG:
4199                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
4200                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
4201                 break;
4202         case HWRM_RING_ALLOC_CMPL:
4203                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
4204                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
4205                 if (bp->flags & BNXT_FLAG_USING_MSIX)
4206                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
4207                 break;
4208         default:
4209                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
4210                            ring_type);
4211                 return -1;
4212         }
4213
4214         mutex_lock(&bp->hwrm_cmd_lock);
4215         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4216         err = le16_to_cpu(resp->error_code);
4217         ring_id = le16_to_cpu(resp->ring_id);
4218         mutex_unlock(&bp->hwrm_cmd_lock);
4219
4220         if (rc || err) {
4221                 switch (ring_type) {
4222                 case RING_FREE_REQ_RING_TYPE_L2_CMPL:
4223                         netdev_err(bp->dev, "hwrm_ring_alloc cp failed. rc:%x err:%x\n",
4224                                    rc, err);
4225                         return -1;
4226
4227                 case RING_FREE_REQ_RING_TYPE_RX:
4228                         netdev_err(bp->dev, "hwrm_ring_alloc rx failed. rc:%x err:%x\n",
4229                                    rc, err);
4230                         return -1;
4231
4232                 case RING_FREE_REQ_RING_TYPE_TX:
4233                         netdev_err(bp->dev, "hwrm_ring_alloc tx failed. rc:%x err:%x\n",
4234                                    rc, err);
4235                         return -1;
4236
4237                 default:
4238                         netdev_err(bp->dev, "Invalid ring\n");
4239                         return -1;
4240                 }
4241         }
4242         ring->fw_ring_id = ring_id;
4243         return rc;
4244 }
4245
4246 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
4247 {
4248         int rc;
4249
4250         if (BNXT_PF(bp)) {
4251                 struct hwrm_func_cfg_input req = {0};
4252
4253                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
4254                 req.fid = cpu_to_le16(0xffff);
4255                 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
4256                 req.async_event_cr = cpu_to_le16(idx);
4257                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4258         } else {
4259                 struct hwrm_func_vf_cfg_input req = {0};
4260
4261                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
4262                 req.enables =
4263                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
4264                 req.async_event_cr = cpu_to_le16(idx);
4265                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4266         }
4267         return rc;
4268 }
4269
4270 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
4271 {
4272         int i, rc = 0;
4273
4274         for (i = 0; i < bp->cp_nr_rings; i++) {
4275                 struct bnxt_napi *bnapi = bp->bnapi[i];
4276                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4277                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4278
4279                 cpr->cp_doorbell = bp->bar1 + i * 0x80;
4280                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL, i,
4281                                               INVALID_STATS_CTX_ID);
4282                 if (rc)
4283                         goto err_out;
4284                 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
4285                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
4286
4287                 if (!i) {
4288                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
4289                         if (rc)
4290                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
4291                 }
4292         }
4293
4294         for (i = 0; i < bp->tx_nr_rings; i++) {
4295                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4296                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
4297                 u32 map_idx = txr->bnapi->index;
4298                 u16 fw_stats_ctx = bp->grp_info[map_idx].fw_stats_ctx;
4299
4300                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX,
4301                                               map_idx, fw_stats_ctx);
4302                 if (rc)
4303                         goto err_out;
4304                 txr->tx_doorbell = bp->bar1 + map_idx * 0x80;
4305         }
4306
4307         for (i = 0; i < bp->rx_nr_rings; i++) {
4308                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4309                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
4310                 u32 map_idx = rxr->bnapi->index;
4311
4312                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX,
4313                                               map_idx, INVALID_STATS_CTX_ID);
4314                 if (rc)
4315                         goto err_out;
4316                 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80;
4317                 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
4318                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
4319         }
4320
4321         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
4322                 for (i = 0; i < bp->rx_nr_rings; i++) {
4323                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4324                         struct bnxt_ring_struct *ring =
4325                                                 &rxr->rx_agg_ring_struct;
4326                         u32 grp_idx = rxr->bnapi->index;
4327                         u32 map_idx = grp_idx + bp->rx_nr_rings;
4328
4329                         rc = hwrm_ring_alloc_send_msg(bp, ring,
4330                                                       HWRM_RING_ALLOC_AGG,
4331                                                       map_idx,
4332                                                       INVALID_STATS_CTX_ID);
4333                         if (rc)
4334                                 goto err_out;
4335
4336                         rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80;
4337                         writel(DB_KEY_RX | rxr->rx_agg_prod,
4338                                rxr->rx_agg_doorbell);
4339                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
4340                 }
4341         }
4342 err_out:
4343         return rc;
4344 }
4345
4346 static int hwrm_ring_free_send_msg(struct bnxt *bp,
4347                                    struct bnxt_ring_struct *ring,
4348                                    u32 ring_type, int cmpl_ring_id)
4349 {
4350         int rc;
4351         struct hwrm_ring_free_input req = {0};
4352         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
4353         u16 error_code;
4354
4355         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
4356         req.ring_type = ring_type;
4357         req.ring_id = cpu_to_le16(ring->fw_ring_id);
4358
4359         mutex_lock(&bp->hwrm_cmd_lock);
4360         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4361         error_code = le16_to_cpu(resp->error_code);
4362         mutex_unlock(&bp->hwrm_cmd_lock);
4363
4364         if (rc || error_code) {
4365                 switch (ring_type) {
4366                 case RING_FREE_REQ_RING_TYPE_L2_CMPL:
4367                         netdev_err(bp->dev, "hwrm_ring_free cp failed. rc:%d\n",
4368                                    rc);
4369                         return rc;
4370                 case RING_FREE_REQ_RING_TYPE_RX:
4371                         netdev_err(bp->dev, "hwrm_ring_free rx failed. rc:%d\n",
4372                                    rc);
4373                         return rc;
4374                 case RING_FREE_REQ_RING_TYPE_TX:
4375                         netdev_err(bp->dev, "hwrm_ring_free tx failed. rc:%d\n",
4376                                    rc);
4377                         return rc;
4378                 default:
4379                         netdev_err(bp->dev, "Invalid ring\n");
4380                         return -1;
4381                 }
4382         }
4383         return 0;
4384 }
4385
4386 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
4387 {
4388         int i;
4389
4390         if (!bp->bnapi)
4391                 return;
4392
4393         for (i = 0; i < bp->tx_nr_rings; i++) {
4394                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4395                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
4396                 u32 grp_idx = txr->bnapi->index;
4397                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
4398
4399                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4400                         hwrm_ring_free_send_msg(bp, ring,
4401                                                 RING_FREE_REQ_RING_TYPE_TX,
4402                                                 close_path ? cmpl_ring_id :
4403                                                 INVALID_HW_RING_ID);
4404                         ring->fw_ring_id = INVALID_HW_RING_ID;
4405                 }
4406         }
4407
4408         for (i = 0; i < bp->rx_nr_rings; i++) {
4409                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4410                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
4411                 u32 grp_idx = rxr->bnapi->index;
4412                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
4413
4414                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4415                         hwrm_ring_free_send_msg(bp, ring,
4416                                                 RING_FREE_REQ_RING_TYPE_RX,
4417                                                 close_path ? cmpl_ring_id :
4418                                                 INVALID_HW_RING_ID);
4419                         ring->fw_ring_id = INVALID_HW_RING_ID;
4420                         bp->grp_info[grp_idx].rx_fw_ring_id =
4421                                 INVALID_HW_RING_ID;
4422                 }
4423         }
4424
4425         for (i = 0; i < bp->rx_nr_rings; i++) {
4426                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4427                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
4428                 u32 grp_idx = rxr->bnapi->index;
4429                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
4430
4431                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4432                         hwrm_ring_free_send_msg(bp, ring,
4433                                                 RING_FREE_REQ_RING_TYPE_RX,
4434                                                 close_path ? cmpl_ring_id :
4435                                                 INVALID_HW_RING_ID);
4436                         ring->fw_ring_id = INVALID_HW_RING_ID;
4437                         bp->grp_info[grp_idx].agg_fw_ring_id =
4438                                 INVALID_HW_RING_ID;
4439                 }
4440         }
4441
4442         /* The completion rings are about to be freed.  After that the
4443          * IRQ doorbell will not work anymore.  So we need to disable
4444          * IRQ here.
4445          */
4446         bnxt_disable_int_sync(bp);
4447
4448         for (i = 0; i < bp->cp_nr_rings; i++) {
4449                 struct bnxt_napi *bnapi = bp->bnapi[i];
4450                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4451                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4452
4453                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4454                         hwrm_ring_free_send_msg(bp, ring,
4455                                                 RING_FREE_REQ_RING_TYPE_L2_CMPL,
4456                                                 INVALID_HW_RING_ID);
4457                         ring->fw_ring_id = INVALID_HW_RING_ID;
4458                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
4459                 }
4460         }
4461 }
4462
4463 /* Caller must hold bp->hwrm_cmd_lock */
4464 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
4465 {
4466         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
4467         struct hwrm_func_qcfg_input req = {0};
4468         int rc;
4469
4470         if (bp->hwrm_spec_code < 0x10601)
4471                 return 0;
4472
4473         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
4474         req.fid = cpu_to_le16(fid);
4475         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4476         if (!rc)
4477                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
4478
4479         return rc;
4480 }
4481
4482 static int bnxt_hwrm_reserve_tx_rings(struct bnxt *bp, int *tx_rings)
4483 {
4484         struct hwrm_func_cfg_input req = {0};
4485         int rc;
4486
4487         if (bp->hwrm_spec_code < 0x10601)
4488                 return 0;
4489
4490         if (BNXT_VF(bp))
4491                 return 0;
4492
4493         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
4494         req.fid = cpu_to_le16(0xffff);
4495         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS);
4496         req.num_tx_rings = cpu_to_le16(*tx_rings);
4497         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4498         if (rc)
4499                 return rc;
4500
4501         mutex_lock(&bp->hwrm_cmd_lock);
4502         rc = __bnxt_hwrm_get_tx_rings(bp, 0xffff, tx_rings);
4503         mutex_unlock(&bp->hwrm_cmd_lock);
4504         if (!rc)
4505                 bp->tx_reserved_rings = *tx_rings;
4506         return rc;
4507 }
4508
4509 static int bnxt_hwrm_check_tx_rings(struct bnxt *bp, int tx_rings)
4510 {
4511         struct hwrm_func_cfg_input req = {0};
4512         int rc;
4513
4514         if (bp->hwrm_spec_code < 0x10801)
4515                 return 0;
4516
4517         if (BNXT_VF(bp))
4518                 return 0;
4519
4520         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
4521         req.fid = cpu_to_le16(0xffff);
4522         req.flags = cpu_to_le32(FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST);
4523         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS);
4524         req.num_tx_rings = cpu_to_le16(tx_rings);
4525         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4526         if (rc)
4527                 return -ENOMEM;
4528         return 0;
4529 }
4530
4531 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, u32 max_bufs,
4532         u32 buf_tmrs, u16 flags,
4533         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
4534 {
4535         req->flags = cpu_to_le16(flags);
4536         req->num_cmpl_dma_aggr = cpu_to_le16((u16)max_bufs);
4537         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(max_bufs >> 16);
4538         req->cmpl_aggr_dma_tmr = cpu_to_le16((u16)buf_tmrs);
4539         req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(buf_tmrs >> 16);
4540         /* Minimum time between 2 interrupts set to buf_tmr x 2 */
4541         req->int_lat_tmr_min = cpu_to_le16((u16)buf_tmrs * 2);
4542         req->int_lat_tmr_max = cpu_to_le16((u16)buf_tmrs * 4);
4543         req->num_cmpl_aggr_int = cpu_to_le16((u16)max_bufs * 4);
4544 }
4545
4546 int bnxt_hwrm_set_coal(struct bnxt *bp)
4547 {
4548         int i, rc = 0;
4549         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
4550                                                            req_tx = {0}, *req;
4551         u16 max_buf, max_buf_irq;
4552         u16 buf_tmr, buf_tmr_irq;
4553         u32 flags;
4554
4555         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
4556                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
4557         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
4558                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
4559
4560         /* Each rx completion (2 records) should be DMAed immediately.
4561          * DMA 1/4 of the completion buffers at a time.
4562          */
4563         max_buf = min_t(u16, bp->rx_coal_bufs / 4, 2);
4564         /* max_buf must not be zero */
4565         max_buf = clamp_t(u16, max_buf, 1, 63);
4566         max_buf_irq = clamp_t(u16, bp->rx_coal_bufs_irq, 1, 63);
4567         buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks);
4568         /* buf timer set to 1/4 of interrupt timer */
4569         buf_tmr = max_t(u16, buf_tmr / 4, 1);
4570         buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks_irq);
4571         buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
4572
4573         flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
4574
4575         /* RING_IDLE generates more IRQs for lower latency.  Enable it only
4576          * if coal_ticks is less than 25 us.
4577          */
4578         if (bp->rx_coal_ticks < 25)
4579                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
4580
4581         bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
4582                                   buf_tmr_irq << 16 | buf_tmr, flags, &req_rx);
4583
4584         /* max_buf must not be zero */
4585         max_buf = clamp_t(u16, bp->tx_coal_bufs, 1, 63);
4586         max_buf_irq = clamp_t(u16, bp->tx_coal_bufs_irq, 1, 63);
4587         buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks);
4588         /* buf timer set to 1/4 of interrupt timer */
4589         buf_tmr = max_t(u16, buf_tmr / 4, 1);
4590         buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks_irq);
4591         buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
4592
4593         flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
4594         bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
4595                                   buf_tmr_irq << 16 | buf_tmr, flags, &req_tx);
4596
4597         mutex_lock(&bp->hwrm_cmd_lock);
4598         for (i = 0; i < bp->cp_nr_rings; i++) {
4599                 struct bnxt_napi *bnapi = bp->bnapi[i];
4600
4601                 req = &req_rx;
4602                 if (!bnapi->rx_ring)
4603                         req = &req_tx;
4604                 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id);
4605
4606                 rc = _hwrm_send_message(bp, req, sizeof(*req),
4607                                         HWRM_CMD_TIMEOUT);
4608                 if (rc)
4609                         break;
4610         }
4611         mutex_unlock(&bp->hwrm_cmd_lock);
4612         return rc;
4613 }
4614
4615 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
4616 {
4617         int rc = 0, i;
4618         struct hwrm_stat_ctx_free_input req = {0};
4619
4620         if (!bp->bnapi)
4621                 return 0;
4622
4623         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4624                 return 0;
4625
4626         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
4627
4628         mutex_lock(&bp->hwrm_cmd_lock);
4629         for (i = 0; i < bp->cp_nr_rings; i++) {
4630                 struct bnxt_napi *bnapi = bp->bnapi[i];
4631                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4632
4633                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
4634                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
4635
4636                         rc = _hwrm_send_message(bp, &req, sizeof(req),
4637                                                 HWRM_CMD_TIMEOUT);
4638                         if (rc)
4639                                 break;
4640
4641                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4642                 }
4643         }
4644         mutex_unlock(&bp->hwrm_cmd_lock);
4645         return rc;
4646 }
4647
4648 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
4649 {
4650         int rc = 0, i;
4651         struct hwrm_stat_ctx_alloc_input req = {0};
4652         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4653
4654         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4655                 return 0;
4656
4657         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
4658
4659         req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
4660
4661         mutex_lock(&bp->hwrm_cmd_lock);
4662         for (i = 0; i < bp->cp_nr_rings; i++) {
4663                 struct bnxt_napi *bnapi = bp->bnapi[i];
4664                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4665
4666                 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
4667
4668                 rc = _hwrm_send_message(bp, &req, sizeof(req),
4669                                         HWRM_CMD_TIMEOUT);
4670                 if (rc)
4671                         break;
4672
4673                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
4674
4675                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
4676         }
4677         mutex_unlock(&bp->hwrm_cmd_lock);
4678         return rc;
4679 }
4680
4681 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
4682 {
4683         struct hwrm_func_qcfg_input req = {0};
4684         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
4685         u16 flags;
4686         int rc;
4687
4688         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
4689         req.fid = cpu_to_le16(0xffff);
4690         mutex_lock(&bp->hwrm_cmd_lock);
4691         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4692         if (rc)
4693                 goto func_qcfg_exit;
4694
4695 #ifdef CONFIG_BNXT_SRIOV
4696         if (BNXT_VF(bp)) {
4697                 struct bnxt_vf_info *vf = &bp->vf;
4698
4699                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
4700         }
4701 #endif
4702         flags = le16_to_cpu(resp->flags);
4703         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
4704                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
4705                 bp->flags |= BNXT_FLAG_FW_LLDP_AGENT;
4706                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
4707                         bp->flags |= BNXT_FLAG_FW_DCBX_AGENT;
4708         }
4709         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
4710                 bp->flags |= BNXT_FLAG_MULTI_HOST;
4711
4712         switch (resp->port_partition_type) {
4713         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
4714         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
4715         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
4716                 bp->port_partition_type = resp->port_partition_type;
4717                 break;
4718         }
4719         if (bp->hwrm_spec_code < 0x10707 ||
4720             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
4721                 bp->br_mode = BRIDGE_MODE_VEB;
4722         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
4723                 bp->br_mode = BRIDGE_MODE_VEPA;
4724         else
4725                 bp->br_mode = BRIDGE_MODE_UNDEF;
4726
4727 func_qcfg_exit:
4728         mutex_unlock(&bp->hwrm_cmd_lock);
4729         return rc;
4730 }
4731
4732 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
4733 {
4734         int rc = 0;
4735         struct hwrm_func_qcaps_input req = {0};
4736         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4737
4738         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
4739         req.fid = cpu_to_le16(0xffff);
4740
4741         mutex_lock(&bp->hwrm_cmd_lock);
4742         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4743         if (rc)
4744                 goto hwrm_func_qcaps_exit;
4745
4746         if (resp->flags & cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED))
4747                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
4748         if (resp->flags & cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED))
4749                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
4750
4751         bp->tx_push_thresh = 0;
4752         if (resp->flags &
4753             cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED))
4754                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
4755
4756         if (BNXT_PF(bp)) {
4757                 struct bnxt_pf_info *pf = &bp->pf;
4758
4759                 pf->fw_fid = le16_to_cpu(resp->fid);
4760                 pf->port_id = le16_to_cpu(resp->port_id);
4761                 bp->dev->dev_port = pf->port_id;
4762                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
4763                 pf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
4764                 pf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
4765                 pf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
4766                 pf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
4767                 pf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
4768                 if (!pf->max_hw_ring_grps)
4769                         pf->max_hw_ring_grps = pf->max_tx_rings;
4770                 pf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
4771                 pf->max_vnics = le16_to_cpu(resp->max_vnics);
4772                 pf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
4773                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
4774                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
4775                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
4776                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
4777                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
4778                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
4779                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
4780                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
4781                 if (resp->flags &
4782                     cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED))
4783                         bp->flags |= BNXT_FLAG_WOL_CAP;
4784         } else {
4785 #ifdef CONFIG_BNXT_SRIOV
4786                 struct bnxt_vf_info *vf = &bp->vf;
4787
4788                 vf->fw_fid = le16_to_cpu(resp->fid);
4789
4790                 vf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
4791                 vf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
4792                 vf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
4793                 vf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
4794                 vf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
4795                 if (!vf->max_hw_ring_grps)
4796                         vf->max_hw_ring_grps = vf->max_tx_rings;
4797                 vf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
4798                 vf->max_vnics = le16_to_cpu(resp->max_vnics);
4799                 vf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
4800
4801                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
4802 #endif
4803         }
4804
4805 hwrm_func_qcaps_exit:
4806         mutex_unlock(&bp->hwrm_cmd_lock);
4807         return rc;
4808 }
4809
4810 static int bnxt_hwrm_func_reset(struct bnxt *bp)
4811 {
4812         struct hwrm_func_reset_input req = {0};
4813
4814         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
4815         req.enables = 0;
4816
4817         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
4818 }
4819
4820 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
4821 {
4822         int rc = 0;
4823         struct hwrm_queue_qportcfg_input req = {0};
4824         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
4825         u8 i, *qptr;
4826
4827         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
4828
4829         mutex_lock(&bp->hwrm_cmd_lock);
4830         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4831         if (rc)
4832                 goto qportcfg_exit;
4833
4834         if (!resp->max_configurable_queues) {
4835                 rc = -EINVAL;
4836                 goto qportcfg_exit;
4837         }
4838         bp->max_tc = resp->max_configurable_queues;
4839         bp->max_lltc = resp->max_configurable_lossless_queues;
4840         if (bp->max_tc > BNXT_MAX_QUEUE)
4841                 bp->max_tc = BNXT_MAX_QUEUE;
4842
4843         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
4844                 bp->max_tc = 1;
4845
4846         if (bp->max_lltc > bp->max_tc)
4847                 bp->max_lltc = bp->max_tc;
4848
4849         qptr = &resp->queue_id0;
4850         for (i = 0; i < bp->max_tc; i++) {
4851                 bp->q_info[i].queue_id = *qptr++;
4852                 bp->q_info[i].queue_profile = *qptr++;
4853         }
4854
4855 qportcfg_exit:
4856         mutex_unlock(&bp->hwrm_cmd_lock);
4857         return rc;
4858 }
4859
4860 static int bnxt_hwrm_ver_get(struct bnxt *bp)
4861 {
4862         int rc;
4863         struct hwrm_ver_get_input req = {0};
4864         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
4865         u32 dev_caps_cfg;
4866
4867         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
4868         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
4869         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
4870         req.hwrm_intf_min = HWRM_VERSION_MINOR;
4871         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
4872         mutex_lock(&bp->hwrm_cmd_lock);
4873         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4874         if (rc)
4875                 goto hwrm_ver_get_exit;
4876
4877         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
4878
4879         bp->hwrm_spec_code = resp->hwrm_intf_maj << 16 |
4880                              resp->hwrm_intf_min << 8 | resp->hwrm_intf_upd;
4881         if (resp->hwrm_intf_maj < 1) {
4882                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
4883                             resp->hwrm_intf_maj, resp->hwrm_intf_min,
4884                             resp->hwrm_intf_upd);
4885                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
4886         }
4887         snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d/%d.%d.%d",
4888                  resp->hwrm_fw_maj, resp->hwrm_fw_min, resp->hwrm_fw_bld,
4889                  resp->hwrm_intf_maj, resp->hwrm_intf_min, resp->hwrm_intf_upd);
4890
4891         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
4892         if (!bp->hwrm_cmd_timeout)
4893                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
4894
4895         if (resp->hwrm_intf_maj >= 1)
4896                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
4897
4898         bp->chip_num = le16_to_cpu(resp->chip_num);
4899         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
4900             !resp->chip_metal)
4901                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
4902
4903         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
4904         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
4905             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
4906                 bp->flags |= BNXT_FLAG_SHORT_CMD;
4907
4908 hwrm_ver_get_exit:
4909         mutex_unlock(&bp->hwrm_cmd_lock);
4910         return rc;
4911 }
4912
4913 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
4914 {
4915 #if IS_ENABLED(CONFIG_RTC_LIB)
4916         struct hwrm_fw_set_time_input req = {0};
4917         struct rtc_time tm;
4918         struct timeval tv;
4919
4920         if (bp->hwrm_spec_code < 0x10400)
4921                 return -EOPNOTSUPP;
4922
4923         do_gettimeofday(&tv);
4924         rtc_time_to_tm(tv.tv_sec, &tm);
4925         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
4926         req.year = cpu_to_le16(1900 + tm.tm_year);
4927         req.month = 1 + tm.tm_mon;
4928         req.day = tm.tm_mday;
4929         req.hour = tm.tm_hour;
4930         req.minute = tm.tm_min;
4931         req.second = tm.tm_sec;
4932         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4933 #else
4934         return -EOPNOTSUPP;
4935 #endif
4936 }
4937
4938 static int bnxt_hwrm_port_qstats(struct bnxt *bp)
4939 {
4940         int rc;
4941         struct bnxt_pf_info *pf = &bp->pf;
4942         struct hwrm_port_qstats_input req = {0};
4943
4944         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
4945                 return 0;
4946
4947         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
4948         req.port_id = cpu_to_le16(pf->port_id);
4949         req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
4950         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
4951         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4952         return rc;
4953 }
4954
4955 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
4956 {
4957         if (bp->vxlan_port_cnt) {
4958                 bnxt_hwrm_tunnel_dst_port_free(
4959                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
4960         }
4961         bp->vxlan_port_cnt = 0;
4962         if (bp->nge_port_cnt) {
4963                 bnxt_hwrm_tunnel_dst_port_free(
4964                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
4965         }
4966         bp->nge_port_cnt = 0;
4967 }
4968
4969 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
4970 {
4971         int rc, i;
4972         u32 tpa_flags = 0;
4973
4974         if (set_tpa)
4975                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
4976         for (i = 0; i < bp->nr_vnics; i++) {
4977                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
4978                 if (rc) {
4979                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
4980                                    i, rc);
4981                         return rc;
4982                 }
4983         }
4984         return 0;
4985 }
4986
4987 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
4988 {
4989         int i;
4990
4991         for (i = 0; i < bp->nr_vnics; i++)
4992                 bnxt_hwrm_vnic_set_rss(bp, i, false);
4993 }
4994
4995 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
4996                                     bool irq_re_init)
4997 {
4998         if (bp->vnic_info) {
4999                 bnxt_hwrm_clear_vnic_filter(bp);
5000                 /* clear all RSS setting before free vnic ctx */
5001                 bnxt_hwrm_clear_vnic_rss(bp);
5002                 bnxt_hwrm_vnic_ctx_free(bp);
5003                 /* before free the vnic, undo the vnic tpa settings */
5004                 if (bp->flags & BNXT_FLAG_TPA)
5005                         bnxt_set_tpa(bp, false);
5006                 bnxt_hwrm_vnic_free(bp);
5007         }
5008         bnxt_hwrm_ring_free(bp, close_path);
5009         bnxt_hwrm_ring_grp_free(bp);
5010         if (irq_re_init) {
5011                 bnxt_hwrm_stat_ctx_free(bp);
5012                 bnxt_hwrm_free_tunnel_ports(bp);
5013         }
5014 }
5015
5016 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
5017 {
5018         struct hwrm_func_cfg_input req = {0};
5019         int rc;
5020
5021         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5022         req.fid = cpu_to_le16(0xffff);
5023         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
5024         if (br_mode == BRIDGE_MODE_VEB)
5025                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
5026         else if (br_mode == BRIDGE_MODE_VEPA)
5027                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
5028         else
5029                 return -EINVAL;
5030         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5031         if (rc)
5032                 rc = -EIO;
5033         return rc;
5034 }
5035
5036 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
5037 {
5038         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5039         int rc;
5040
5041         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
5042                 goto skip_rss_ctx;
5043
5044         /* allocate context for vnic */
5045         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
5046         if (rc) {
5047                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
5048                            vnic_id, rc);
5049                 goto vnic_setup_err;
5050         }
5051         bp->rsscos_nr_ctxs++;
5052
5053         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
5054                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
5055                 if (rc) {
5056                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
5057                                    vnic_id, rc);
5058                         goto vnic_setup_err;
5059                 }
5060                 bp->rsscos_nr_ctxs++;
5061         }
5062
5063 skip_rss_ctx:
5064         /* configure default vnic, ring grp */
5065         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
5066         if (rc) {
5067                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
5068                            vnic_id, rc);
5069                 goto vnic_setup_err;
5070         }
5071
5072         /* Enable RSS hashing on vnic */
5073         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
5074         if (rc) {
5075                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
5076                            vnic_id, rc);
5077                 goto vnic_setup_err;
5078         }
5079
5080         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
5081                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
5082                 if (rc) {
5083                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
5084                                    vnic_id, rc);
5085                 }
5086         }
5087
5088 vnic_setup_err:
5089         return rc;
5090 }
5091
5092 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
5093 {
5094 #ifdef CONFIG_RFS_ACCEL
5095         int i, rc = 0;
5096
5097         for (i = 0; i < bp->rx_nr_rings; i++) {
5098                 struct bnxt_vnic_info *vnic;
5099                 u16 vnic_id = i + 1;
5100                 u16 ring_id = i;
5101
5102                 if (vnic_id >= bp->nr_vnics)
5103                         break;
5104
5105                 vnic = &bp->vnic_info[vnic_id];
5106                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
5107                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
5108                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
5109                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
5110                 if (rc) {
5111                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
5112                                    vnic_id, rc);
5113                         break;
5114                 }
5115                 rc = bnxt_setup_vnic(bp, vnic_id);
5116                 if (rc)
5117                         break;
5118         }
5119         return rc;
5120 #else
5121         return 0;
5122 #endif
5123 }
5124
5125 /* Allow PF and VF with default VLAN to be in promiscuous mode */
5126 static bool bnxt_promisc_ok(struct bnxt *bp)
5127 {
5128 #ifdef CONFIG_BNXT_SRIOV
5129         if (BNXT_VF(bp) && !bp->vf.vlan)
5130                 return false;
5131 #endif
5132         return true;
5133 }
5134
5135 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
5136 {
5137         unsigned int rc = 0;
5138
5139         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
5140         if (rc) {
5141                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
5142                            rc);
5143                 return rc;
5144         }
5145
5146         rc = bnxt_hwrm_vnic_cfg(bp, 1);
5147         if (rc) {
5148                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
5149                            rc);
5150                 return rc;
5151         }
5152         return rc;
5153 }
5154
5155 static int bnxt_cfg_rx_mode(struct bnxt *);
5156 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
5157
5158 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
5159 {
5160         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5161         int rc = 0;
5162         unsigned int rx_nr_rings = bp->rx_nr_rings;
5163
5164         if (irq_re_init) {
5165                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
5166                 if (rc) {
5167                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
5168                                    rc);
5169                         goto err_out;
5170                 }
5171                 if (bp->tx_reserved_rings != bp->tx_nr_rings) {
5172                         int tx = bp->tx_nr_rings;
5173
5174                         if (bnxt_hwrm_reserve_tx_rings(bp, &tx) ||
5175                             tx < bp->tx_nr_rings) {
5176                                 rc = -ENOMEM;
5177                                 goto err_out;
5178                         }
5179                 }
5180         }
5181
5182         rc = bnxt_hwrm_ring_alloc(bp);
5183         if (rc) {
5184                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
5185                 goto err_out;
5186         }
5187
5188         rc = bnxt_hwrm_ring_grp_alloc(bp);
5189         if (rc) {
5190                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
5191                 goto err_out;
5192         }
5193
5194         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5195                 rx_nr_rings--;
5196
5197         /* default vnic 0 */
5198         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
5199         if (rc) {
5200                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
5201                 goto err_out;
5202         }
5203
5204         rc = bnxt_setup_vnic(bp, 0);
5205         if (rc)
5206                 goto err_out;
5207
5208         if (bp->flags & BNXT_FLAG_RFS) {
5209                 rc = bnxt_alloc_rfs_vnics(bp);
5210                 if (rc)
5211                         goto err_out;
5212         }
5213
5214         if (bp->flags & BNXT_FLAG_TPA) {
5215                 rc = bnxt_set_tpa(bp, true);
5216                 if (rc)
5217                         goto err_out;
5218         }
5219
5220         if (BNXT_VF(bp))
5221                 bnxt_update_vf_mac(bp);
5222
5223         /* Filter for default vnic 0 */
5224         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
5225         if (rc) {
5226                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
5227                 goto err_out;
5228         }
5229         vnic->uc_filter_count = 1;
5230
5231         vnic->rx_mask = CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
5232
5233         if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
5234                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5235
5236         if (bp->dev->flags & IFF_ALLMULTI) {
5237                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5238                 vnic->mc_list_count = 0;
5239         } else {
5240                 u32 mask = 0;
5241
5242                 bnxt_mc_list_updated(bp, &mask);
5243                 vnic->rx_mask |= mask;
5244         }
5245
5246         rc = bnxt_cfg_rx_mode(bp);
5247         if (rc)
5248                 goto err_out;
5249
5250         rc = bnxt_hwrm_set_coal(bp);
5251         if (rc)
5252                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
5253                                 rc);
5254
5255         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
5256                 rc = bnxt_setup_nitroa0_vnic(bp);
5257                 if (rc)
5258                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
5259                                    rc);
5260         }
5261
5262         if (BNXT_VF(bp)) {
5263                 bnxt_hwrm_func_qcfg(bp);
5264                 netdev_update_features(bp->dev);
5265         }
5266
5267         return 0;
5268
5269 err_out:
5270         bnxt_hwrm_resource_free(bp, 0, true);
5271
5272         return rc;
5273 }
5274
5275 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
5276 {
5277         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
5278         return 0;
5279 }
5280
5281 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
5282 {
5283         bnxt_init_cp_rings(bp);
5284         bnxt_init_rx_rings(bp);
5285         bnxt_init_tx_rings(bp);
5286         bnxt_init_ring_grps(bp, irq_re_init);
5287         bnxt_init_vnics(bp);
5288
5289         return bnxt_init_chip(bp, irq_re_init);
5290 }
5291
5292 static int bnxt_set_real_num_queues(struct bnxt *bp)
5293 {
5294         int rc;
5295         struct net_device *dev = bp->dev;
5296
5297         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
5298                                           bp->tx_nr_rings_xdp);
5299         if (rc)
5300                 return rc;
5301
5302         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
5303         if (rc)
5304                 return rc;
5305
5306 #ifdef CONFIG_RFS_ACCEL
5307         if (bp->flags & BNXT_FLAG_RFS)
5308                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
5309 #endif
5310
5311         return rc;
5312 }
5313
5314 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5315                            bool shared)
5316 {
5317         int _rx = *rx, _tx = *tx;
5318
5319         if (shared) {
5320                 *rx = min_t(int, _rx, max);
5321                 *tx = min_t(int, _tx, max);
5322         } else {
5323                 if (max < 2)
5324                         return -ENOMEM;
5325
5326                 while (_rx + _tx > max) {
5327                         if (_rx > _tx && _rx > 1)
5328                                 _rx--;
5329                         else if (_tx > 1)
5330                                 _tx--;
5331                 }
5332                 *rx = _rx;
5333                 *tx = _tx;
5334         }
5335         return 0;
5336 }
5337
5338 static void bnxt_setup_msix(struct bnxt *bp)
5339 {
5340         const int len = sizeof(bp->irq_tbl[0].name);
5341         struct net_device *dev = bp->dev;
5342         int tcs, i;
5343
5344         tcs = netdev_get_num_tc(dev);
5345         if (tcs > 1) {
5346                 int i, off, count;
5347
5348                 for (i = 0; i < tcs; i++) {
5349                         count = bp->tx_nr_rings_per_tc;
5350                         off = i * count;
5351                         netdev_set_tc_queue(dev, i, count, off);
5352                 }
5353         }
5354
5355         for (i = 0; i < bp->cp_nr_rings; i++) {
5356                 char *attr;
5357
5358                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5359                         attr = "TxRx";
5360                 else if (i < bp->rx_nr_rings)
5361                         attr = "rx";
5362                 else
5363                         attr = "tx";
5364
5365                 snprintf(bp->irq_tbl[i].name, len, "%s-%s-%d", dev->name, attr,
5366                          i);
5367                 bp->irq_tbl[i].handler = bnxt_msix;
5368         }
5369 }
5370
5371 static void bnxt_setup_inta(struct bnxt *bp)
5372 {
5373         const int len = sizeof(bp->irq_tbl[0].name);
5374
5375         if (netdev_get_num_tc(bp->dev))
5376                 netdev_reset_tc(bp->dev);
5377
5378         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
5379                  0);
5380         bp->irq_tbl[0].handler = bnxt_inta;
5381 }
5382
5383 static int bnxt_setup_int_mode(struct bnxt *bp)
5384 {
5385         int rc;
5386
5387         if (bp->flags & BNXT_FLAG_USING_MSIX)
5388                 bnxt_setup_msix(bp);
5389         else
5390                 bnxt_setup_inta(bp);
5391
5392         rc = bnxt_set_real_num_queues(bp);
5393         return rc;
5394 }
5395
5396 #ifdef CONFIG_RFS_ACCEL
5397 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
5398 {
5399 #if defined(CONFIG_BNXT_SRIOV)
5400         if (BNXT_VF(bp))
5401                 return bp->vf.max_rsscos_ctxs;
5402 #endif
5403         return bp->pf.max_rsscos_ctxs;
5404 }
5405
5406 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
5407 {
5408 #if defined(CONFIG_BNXT_SRIOV)
5409         if (BNXT_VF(bp))
5410                 return bp->vf.max_vnics;
5411 #endif
5412         return bp->pf.max_vnics;
5413 }
5414 #endif
5415
5416 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
5417 {
5418 #if defined(CONFIG_BNXT_SRIOV)
5419         if (BNXT_VF(bp))
5420                 return bp->vf.max_stat_ctxs;
5421 #endif
5422         return bp->pf.max_stat_ctxs;
5423 }
5424
5425 void bnxt_set_max_func_stat_ctxs(struct bnxt *bp, unsigned int max)
5426 {
5427 #if defined(CONFIG_BNXT_SRIOV)
5428         if (BNXT_VF(bp))
5429                 bp->vf.max_stat_ctxs = max;
5430         else
5431 #endif
5432                 bp->pf.max_stat_ctxs = max;
5433 }
5434
5435 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
5436 {
5437 #if defined(CONFIG_BNXT_SRIOV)
5438         if (BNXT_VF(bp))
5439                 return bp->vf.max_cp_rings;
5440 #endif
5441         return bp->pf.max_cp_rings;
5442 }
5443
5444 void bnxt_set_max_func_cp_rings(struct bnxt *bp, unsigned int max)
5445 {
5446 #if defined(CONFIG_BNXT_SRIOV)
5447         if (BNXT_VF(bp))
5448                 bp->vf.max_cp_rings = max;
5449         else
5450 #endif
5451                 bp->pf.max_cp_rings = max;
5452 }
5453
5454 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
5455 {
5456 #if defined(CONFIG_BNXT_SRIOV)
5457         if (BNXT_VF(bp))
5458                 return min_t(unsigned int, bp->vf.max_irqs,
5459                              bp->vf.max_cp_rings);
5460 #endif
5461         return min_t(unsigned int, bp->pf.max_irqs, bp->pf.max_cp_rings);
5462 }
5463
5464 void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
5465 {
5466 #if defined(CONFIG_BNXT_SRIOV)
5467         if (BNXT_VF(bp))
5468                 bp->vf.max_irqs = max_irqs;
5469         else
5470 #endif
5471                 bp->pf.max_irqs = max_irqs;
5472 }
5473
5474 static int bnxt_init_msix(struct bnxt *bp)
5475 {
5476         int i, total_vecs, rc = 0, min = 1;
5477         struct msix_entry *msix_ent;
5478
5479         total_vecs = bnxt_get_max_func_irqs(bp);
5480         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
5481         if (!msix_ent)
5482                 return -ENOMEM;
5483
5484         for (i = 0; i < total_vecs; i++) {
5485                 msix_ent[i].entry = i;
5486                 msix_ent[i].vector = 0;
5487         }
5488
5489         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
5490                 min = 2;
5491
5492         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
5493         if (total_vecs < 0) {
5494                 rc = -ENODEV;
5495                 goto msix_setup_exit;
5496         }
5497
5498         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
5499         if (bp->irq_tbl) {
5500                 for (i = 0; i < total_vecs; i++)
5501                         bp->irq_tbl[i].vector = msix_ent[i].vector;
5502
5503                 bp->total_irqs = total_vecs;
5504                 /* Trim rings based upon num of vectors allocated */
5505                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
5506                                      total_vecs, min == 1);
5507                 if (rc)
5508                         goto msix_setup_exit;
5509
5510                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
5511                 bp->cp_nr_rings = (min == 1) ?
5512                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
5513                                   bp->tx_nr_rings + bp->rx_nr_rings;
5514
5515         } else {
5516                 rc = -ENOMEM;
5517                 goto msix_setup_exit;
5518         }
5519         bp->flags |= BNXT_FLAG_USING_MSIX;
5520         kfree(msix_ent);
5521         return 0;
5522
5523 msix_setup_exit:
5524         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
5525         kfree(bp->irq_tbl);
5526         bp->irq_tbl = NULL;
5527         pci_disable_msix(bp->pdev);
5528         kfree(msix_ent);
5529         return rc;
5530 }
5531
5532 static int bnxt_init_inta(struct bnxt *bp)
5533 {
5534         bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
5535         if (!bp->irq_tbl)
5536                 return -ENOMEM;
5537
5538         bp->total_irqs = 1;
5539         bp->rx_nr_rings = 1;
5540         bp->tx_nr_rings = 1;
5541         bp->cp_nr_rings = 1;
5542         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
5543         bp->flags |= BNXT_FLAG_SHARED_RINGS;
5544         bp->irq_tbl[0].vector = bp->pdev->irq;
5545         return 0;
5546 }
5547
5548 static int bnxt_init_int_mode(struct bnxt *bp)
5549 {
5550         int rc = 0;
5551
5552         if (bp->flags & BNXT_FLAG_MSIX_CAP)
5553                 rc = bnxt_init_msix(bp);
5554
5555         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
5556                 /* fallback to INTA */
5557                 rc = bnxt_init_inta(bp);
5558         }
5559         return rc;
5560 }
5561
5562 static void bnxt_clear_int_mode(struct bnxt *bp)
5563 {
5564         if (bp->flags & BNXT_FLAG_USING_MSIX)
5565                 pci_disable_msix(bp->pdev);
5566
5567         kfree(bp->irq_tbl);
5568         bp->irq_tbl = NULL;
5569         bp->flags &= ~BNXT_FLAG_USING_MSIX;
5570 }
5571
5572 static void bnxt_free_irq(struct bnxt *bp)
5573 {
5574         struct bnxt_irq *irq;
5575         int i;
5576
5577 #ifdef CONFIG_RFS_ACCEL
5578         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
5579         bp->dev->rx_cpu_rmap = NULL;
5580 #endif
5581         if (!bp->irq_tbl)
5582                 return;
5583
5584         for (i = 0; i < bp->cp_nr_rings; i++) {
5585                 irq = &bp->irq_tbl[i];
5586                 if (irq->requested) {
5587                         if (irq->have_cpumask) {
5588                                 irq_set_affinity_hint(irq->vector, NULL);
5589                                 free_cpumask_var(irq->cpu_mask);
5590                                 irq->have_cpumask = 0;
5591                         }
5592                         free_irq(irq->vector, bp->bnapi[i]);
5593                 }
5594
5595                 irq->requested = 0;
5596         }
5597 }
5598
5599 static int bnxt_request_irq(struct bnxt *bp)
5600 {
5601         int i, j, rc = 0;
5602         unsigned long flags = 0;
5603 #ifdef CONFIG_RFS_ACCEL
5604         struct cpu_rmap *rmap = bp->dev->rx_cpu_rmap;
5605 #endif
5606
5607         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
5608                 flags = IRQF_SHARED;
5609
5610         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
5611                 struct bnxt_irq *irq = &bp->irq_tbl[i];
5612 #ifdef CONFIG_RFS_ACCEL
5613                 if (rmap && bp->bnapi[i]->rx_ring) {
5614                         rc = irq_cpu_rmap_add(rmap, irq->vector);
5615                         if (rc)
5616                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
5617                                             j);
5618                         j++;
5619                 }
5620 #endif
5621                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
5622                                  bp->bnapi[i]);
5623                 if (rc)
5624                         break;
5625
5626                 irq->requested = 1;
5627
5628                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
5629                         int numa_node = dev_to_node(&bp->pdev->dev);
5630
5631                         irq->have_cpumask = 1;
5632                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
5633                                         irq->cpu_mask);
5634                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
5635                         if (rc) {
5636                                 netdev_warn(bp->dev,
5637                                             "Set affinity failed, IRQ = %d\n",
5638                                             irq->vector);
5639                                 break;
5640                         }
5641                 }
5642         }
5643         return rc;
5644 }
5645
5646 static void bnxt_del_napi(struct bnxt *bp)
5647 {
5648         int i;
5649
5650         if (!bp->bnapi)
5651                 return;
5652
5653         for (i = 0; i < bp->cp_nr_rings; i++) {
5654                 struct bnxt_napi *bnapi = bp->bnapi[i];
5655
5656                 napi_hash_del(&bnapi->napi);
5657                 netif_napi_del(&bnapi->napi);
5658         }
5659         /* We called napi_hash_del() before netif_napi_del(), we need
5660          * to respect an RCU grace period before freeing napi structures.
5661          */
5662         synchronize_net();
5663 }
5664
5665 static void bnxt_init_napi(struct bnxt *bp)
5666 {
5667         int i;
5668         unsigned int cp_nr_rings = bp->cp_nr_rings;
5669         struct bnxt_napi *bnapi;
5670
5671         if (bp->flags & BNXT_FLAG_USING_MSIX) {
5672                 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5673                         cp_nr_rings--;
5674                 for (i = 0; i < cp_nr_rings; i++) {
5675                         bnapi = bp->bnapi[i];
5676                         netif_napi_add(bp->dev, &bnapi->napi,
5677                                        bnxt_poll, 64);
5678                 }
5679                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
5680                         bnapi = bp->bnapi[cp_nr_rings];
5681                         netif_napi_add(bp->dev, &bnapi->napi,
5682                                        bnxt_poll_nitroa0, 64);
5683                 }
5684         } else {
5685                 bnapi = bp->bnapi[0];
5686                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
5687         }
5688 }
5689
5690 static void bnxt_disable_napi(struct bnxt *bp)
5691 {
5692         int i;
5693
5694         if (!bp->bnapi)
5695                 return;
5696
5697         for (i = 0; i < bp->cp_nr_rings; i++)
5698                 napi_disable(&bp->bnapi[i]->napi);
5699 }
5700
5701 static void bnxt_enable_napi(struct bnxt *bp)
5702 {
5703         int i;
5704
5705         for (i = 0; i < bp->cp_nr_rings; i++) {
5706                 bp->bnapi[i]->in_reset = false;
5707                 napi_enable(&bp->bnapi[i]->napi);
5708         }
5709 }
5710
5711 void bnxt_tx_disable(struct bnxt *bp)
5712 {
5713         int i;
5714         struct bnxt_tx_ring_info *txr;
5715
5716         if (bp->tx_ring) {
5717                 for (i = 0; i < bp->tx_nr_rings; i++) {
5718                         txr = &bp->tx_ring[i];
5719                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
5720                 }
5721         }
5722         /* Stop all TX queues */
5723         netif_tx_disable(bp->dev);
5724         netif_carrier_off(bp->dev);
5725 }
5726
5727 void bnxt_tx_enable(struct bnxt *bp)
5728 {
5729         int i;
5730         struct bnxt_tx_ring_info *txr;
5731
5732         for (i = 0; i < bp->tx_nr_rings; i++) {
5733                 txr = &bp->tx_ring[i];
5734                 txr->dev_state = 0;
5735         }
5736         netif_tx_wake_all_queues(bp->dev);
5737         if (bp->link_info.link_up)
5738                 netif_carrier_on(bp->dev);
5739 }
5740
5741 static void bnxt_report_link(struct bnxt *bp)
5742 {
5743         if (bp->link_info.link_up) {
5744                 const char *duplex;
5745                 const char *flow_ctrl;
5746                 u32 speed;
5747                 u16 fec;
5748
5749                 netif_carrier_on(bp->dev);
5750                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
5751                         duplex = "full";
5752                 else
5753                         duplex = "half";
5754                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
5755                         flow_ctrl = "ON - receive & transmit";
5756                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
5757                         flow_ctrl = "ON - transmit";
5758                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
5759                         flow_ctrl = "ON - receive";
5760                 else
5761                         flow_ctrl = "none";
5762                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
5763                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
5764                             speed, duplex, flow_ctrl);
5765                 if (bp->flags & BNXT_FLAG_EEE_CAP)
5766                         netdev_info(bp->dev, "EEE is %s\n",
5767                                     bp->eee.eee_active ? "active" :
5768                                                          "not active");
5769                 fec = bp->link_info.fec_cfg;
5770                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
5771                         netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
5772                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
5773                                     (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
5774                                      (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
5775         } else {
5776                 netif_carrier_off(bp->dev);
5777                 netdev_err(bp->dev, "NIC Link is Down\n");
5778         }
5779 }
5780
5781 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
5782 {
5783         int rc = 0;
5784         struct hwrm_port_phy_qcaps_input req = {0};
5785         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5786         struct bnxt_link_info *link_info = &bp->link_info;
5787
5788         if (bp->hwrm_spec_code < 0x10201)
5789                 return 0;
5790
5791         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
5792
5793         mutex_lock(&bp->hwrm_cmd_lock);
5794         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5795         if (rc)
5796                 goto hwrm_phy_qcaps_exit;
5797
5798         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
5799                 struct ethtool_eee *eee = &bp->eee;
5800                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
5801
5802                 bp->flags |= BNXT_FLAG_EEE_CAP;
5803                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
5804                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
5805                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
5806                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
5807                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
5808         }
5809         if (resp->supported_speeds_auto_mode)
5810                 link_info->support_auto_speeds =
5811                         le16_to_cpu(resp->supported_speeds_auto_mode);
5812
5813         bp->port_count = resp->port_cnt;
5814
5815 hwrm_phy_qcaps_exit:
5816         mutex_unlock(&bp->hwrm_cmd_lock);
5817         return rc;
5818 }
5819
5820 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
5821 {
5822         int rc = 0;
5823         struct bnxt_link_info *link_info = &bp->link_info;
5824         struct hwrm_port_phy_qcfg_input req = {0};
5825         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5826         u8 link_up = link_info->link_up;
5827         u16 diff;
5828
5829         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
5830
5831         mutex_lock(&bp->hwrm_cmd_lock);
5832         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5833         if (rc) {
5834                 mutex_unlock(&bp->hwrm_cmd_lock);
5835                 return rc;
5836         }
5837
5838         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
5839         link_info->phy_link_status = resp->link;
5840         link_info->duplex = resp->duplex_cfg;
5841         if (bp->hwrm_spec_code >= 0x10800)
5842                 link_info->duplex = resp->duplex_state;
5843         link_info->pause = resp->pause;
5844         link_info->auto_mode = resp->auto_mode;
5845         link_info->auto_pause_setting = resp->auto_pause;
5846         link_info->lp_pause = resp->link_partner_adv_pause;
5847         link_info->force_pause_setting = resp->force_pause;
5848         link_info->duplex_setting = resp->duplex_cfg;
5849         if (link_info->phy_link_status == BNXT_LINK_LINK)
5850                 link_info->link_speed = le16_to_cpu(resp->link_speed);
5851         else
5852                 link_info->link_speed = 0;
5853         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
5854         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
5855         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
5856         link_info->lp_auto_link_speeds =
5857                 le16_to_cpu(resp->link_partner_adv_speeds);
5858         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
5859         link_info->phy_ver[0] = resp->phy_maj;
5860         link_info->phy_ver[1] = resp->phy_min;
5861         link_info->phy_ver[2] = resp->phy_bld;
5862         link_info->media_type = resp->media_type;
5863         link_info->phy_type = resp->phy_type;
5864         link_info->transceiver = resp->xcvr_pkg_type;
5865         link_info->phy_addr = resp->eee_config_phy_addr &
5866                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
5867         link_info->module_status = resp->module_status;
5868
5869         if (bp->flags & BNXT_FLAG_EEE_CAP) {
5870                 struct ethtool_eee *eee = &bp->eee;
5871                 u16 fw_speeds;
5872
5873                 eee->eee_active = 0;
5874                 if (resp->eee_config_phy_addr &
5875                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
5876                         eee->eee_active = 1;
5877                         fw_speeds = le16_to_cpu(
5878                                 resp->link_partner_adv_eee_link_speed_mask);
5879                         eee->lp_advertised =
5880                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
5881                 }
5882
5883                 /* Pull initial EEE config */
5884                 if (!chng_link_state) {
5885                         if (resp->eee_config_phy_addr &
5886                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
5887                                 eee->eee_enabled = 1;
5888
5889                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
5890                         eee->advertised =
5891                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
5892
5893                         if (resp->eee_config_phy_addr &
5894                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
5895                                 __le32 tmr;
5896
5897                                 eee->tx_lpi_enabled = 1;
5898                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
5899                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
5900                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
5901                         }
5902                 }
5903         }
5904
5905         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
5906         if (bp->hwrm_spec_code >= 0x10504)
5907                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
5908
5909         /* TODO: need to add more logic to report VF link */
5910         if (chng_link_state) {
5911                 if (link_info->phy_link_status == BNXT_LINK_LINK)
5912                         link_info->link_up = 1;
5913                 else
5914                         link_info->link_up = 0;
5915                 if (link_up != link_info->link_up)
5916                         bnxt_report_link(bp);
5917         } else {
5918                 /* alwasy link down if not require to update link state */
5919                 link_info->link_up = 0;
5920         }
5921         mutex_unlock(&bp->hwrm_cmd_lock);
5922
5923         diff = link_info->support_auto_speeds ^ link_info->advertising;
5924         if ((link_info->support_auto_speeds | diff) !=
5925             link_info->support_auto_speeds) {
5926                 /* An advertised speed is no longer supported, so we need to
5927                  * update the advertisement settings.  Caller holds RTNL
5928                  * so we can modify link settings.
5929                  */
5930                 link_info->advertising = link_info->support_auto_speeds;
5931                 if (link_info->autoneg & BNXT_AUTONEG_SPEED)
5932                         bnxt_hwrm_set_link_setting(bp, true, false);
5933         }
5934         return 0;
5935 }
5936
5937 static void bnxt_get_port_module_status(struct bnxt *bp)
5938 {
5939         struct bnxt_link_info *link_info = &bp->link_info;
5940         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
5941         u8 module_status;
5942
5943         if (bnxt_update_link(bp, true))
5944                 return;
5945
5946         module_status = link_info->module_status;
5947         switch (module_status) {
5948         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
5949         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
5950         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
5951                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
5952                             bp->pf.port_id);
5953                 if (bp->hwrm_spec_code >= 0x10201) {
5954                         netdev_warn(bp->dev, "Module part number %s\n",
5955                                     resp->phy_vendor_partnumber);
5956                 }
5957                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
5958                         netdev_warn(bp->dev, "TX is disabled\n");
5959                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
5960                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
5961         }
5962 }
5963
5964 static void
5965 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
5966 {
5967         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
5968                 if (bp->hwrm_spec_code >= 0x10201)
5969                         req->auto_pause =
5970                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
5971                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
5972                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
5973                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
5974                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
5975                 req->enables |=
5976                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
5977         } else {
5978                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
5979                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
5980                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
5981                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
5982                 req->enables |=
5983                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
5984                 if (bp->hwrm_spec_code >= 0x10201) {
5985                         req->auto_pause = req->force_pause;
5986                         req->enables |= cpu_to_le32(
5987                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
5988                 }
5989         }
5990 }
5991
5992 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
5993                                       struct hwrm_port_phy_cfg_input *req)
5994 {
5995         u8 autoneg = bp->link_info.autoneg;
5996         u16 fw_link_speed = bp->link_info.req_link_speed;
5997         u16 advertising = bp->link_info.advertising;
5998
5999         if (autoneg & BNXT_AUTONEG_SPEED) {
6000                 req->auto_mode |=
6001                         PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
6002
6003                 req->enables |= cpu_to_le32(
6004                         PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
6005                 req->auto_link_speed_mask = cpu_to_le16(advertising);
6006
6007                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
6008                 req->flags |=
6009                         cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
6010         } else {
6011                 req->force_link_speed = cpu_to_le16(fw_link_speed);
6012                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
6013         }
6014
6015         /* tell chimp that the setting takes effect immediately */
6016         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
6017 }
6018
6019 int bnxt_hwrm_set_pause(struct bnxt *bp)
6020 {
6021         struct hwrm_port_phy_cfg_input req = {0};
6022         int rc;
6023
6024         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
6025         bnxt_hwrm_set_pause_common(bp, &req);
6026
6027         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
6028             bp->link_info.force_link_chng)
6029                 bnxt_hwrm_set_link_common(bp, &req);
6030
6031         mutex_lock(&bp->hwrm_cmd_lock);
6032         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6033         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
6034                 /* since changing of pause setting doesn't trigger any link
6035                  * change event, the driver needs to update the current pause
6036                  * result upon successfully return of the phy_cfg command
6037                  */
6038                 bp->link_info.pause =
6039                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
6040                 bp->link_info.auto_pause_setting = 0;
6041                 if (!bp->link_info.force_link_chng)
6042                         bnxt_report_link(bp);
6043         }
6044         bp->link_info.force_link_chng = false;
6045         mutex_unlock(&bp->hwrm_cmd_lock);
6046         return rc;
6047 }
6048
6049 static void bnxt_hwrm_set_eee(struct bnxt *bp,
6050                               struct hwrm_port_phy_cfg_input *req)
6051 {
6052         struct ethtool_eee *eee = &bp->eee;
6053
6054         if (eee->eee_enabled) {
6055                 u16 eee_speeds;
6056                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
6057
6058                 if (eee->tx_lpi_enabled)
6059                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
6060                 else
6061                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
6062
6063                 req->flags |= cpu_to_le32(flags);
6064                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
6065                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
6066                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
6067         } else {
6068                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
6069         }
6070 }
6071
6072 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
6073 {
6074         struct hwrm_port_phy_cfg_input req = {0};
6075
6076         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
6077         if (set_pause)
6078                 bnxt_hwrm_set_pause_common(bp, &req);
6079
6080         bnxt_hwrm_set_link_common(bp, &req);
6081
6082         if (set_eee)
6083                 bnxt_hwrm_set_eee(bp, &req);
6084         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6085 }
6086
6087 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
6088 {
6089         struct hwrm_port_phy_cfg_input req = {0};
6090
6091         if (!BNXT_SINGLE_PF(bp))
6092                 return 0;
6093
6094         if (pci_num_vf(bp->pdev))
6095                 return 0;
6096
6097         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
6098         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
6099         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6100 }
6101
6102 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
6103 {
6104         struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6105         struct hwrm_port_led_qcaps_input req = {0};
6106         struct bnxt_pf_info *pf = &bp->pf;
6107         int rc;
6108
6109         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
6110                 return 0;
6111
6112         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
6113         req.port_id = cpu_to_le16(pf->port_id);
6114         mutex_lock(&bp->hwrm_cmd_lock);
6115         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6116         if (rc) {
6117                 mutex_unlock(&bp->hwrm_cmd_lock);
6118                 return rc;
6119         }
6120         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
6121                 int i;
6122
6123                 bp->num_leds = resp->num_leds;
6124                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
6125                                                  bp->num_leds);
6126                 for (i = 0; i < bp->num_leds; i++) {
6127                         struct bnxt_led_info *led = &bp->leds[i];
6128                         __le16 caps = led->led_state_caps;
6129
6130                         if (!led->led_group_id ||
6131                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
6132                                 bp->num_leds = 0;
6133                                 break;
6134                         }
6135                 }
6136         }
6137         mutex_unlock(&bp->hwrm_cmd_lock);
6138         return 0;
6139 }
6140
6141 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
6142 {
6143         struct hwrm_wol_filter_alloc_input req = {0};
6144         struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6145         int rc;
6146
6147         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
6148         req.port_id = cpu_to_le16(bp->pf.port_id);
6149         req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
6150         req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
6151         memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
6152         mutex_lock(&bp->hwrm_cmd_lock);
6153         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6154         if (!rc)
6155                 bp->wol_filter_id = resp->wol_filter_id;
6156         mutex_unlock(&bp->hwrm_cmd_lock);
6157         return rc;
6158 }
6159
6160 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
6161 {
6162         struct hwrm_wol_filter_free_input req = {0};
6163         int rc;
6164
6165         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
6166         req.port_id = cpu_to_le16(bp->pf.port_id);
6167         req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
6168         req.wol_filter_id = bp->wol_filter_id;
6169         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6170         return rc;
6171 }
6172
6173 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
6174 {
6175         struct hwrm_wol_filter_qcfg_input req = {0};
6176         struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6177         u16 next_handle = 0;
6178         int rc;
6179
6180         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
6181         req.port_id = cpu_to_le16(bp->pf.port_id);
6182         req.handle = cpu_to_le16(handle);
6183         mutex_lock(&bp->hwrm_cmd_lock);
6184         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6185         if (!rc) {
6186                 next_handle = le16_to_cpu(resp->next_handle);
6187                 if (next_handle != 0) {
6188                         if (resp->wol_type ==
6189                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
6190                                 bp->wol = 1;
6191                                 bp->wol_filter_id = resp->wol_filter_id;
6192                         }
6193                 }
6194         }
6195         mutex_unlock(&bp->hwrm_cmd_lock);
6196         return next_handle;
6197 }
6198
6199 static void bnxt_get_wol_settings(struct bnxt *bp)
6200 {
6201         u16 handle = 0;
6202
6203         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
6204                 return;
6205
6206         do {
6207                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
6208         } while (handle && handle != 0xffff);
6209 }
6210
6211 static bool bnxt_eee_config_ok(struct bnxt *bp)
6212 {
6213         struct ethtool_eee *eee = &bp->eee;
6214         struct bnxt_link_info *link_info = &bp->link_info;
6215
6216         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
6217                 return true;
6218
6219         if (eee->eee_enabled) {
6220                 u32 advertising =
6221                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
6222
6223                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
6224                         eee->eee_enabled = 0;
6225                         return false;
6226                 }
6227                 if (eee->advertised & ~advertising) {
6228                         eee->advertised = advertising & eee->supported;
6229                         return false;
6230                 }
6231         }
6232         return true;
6233 }
6234
6235 static int bnxt_update_phy_setting(struct bnxt *bp)
6236 {
6237         int rc;
6238         bool update_link = false;
6239         bool update_pause = false;
6240         bool update_eee = false;
6241         struct bnxt_link_info *link_info = &bp->link_info;
6242
6243         rc = bnxt_update_link(bp, true);
6244         if (rc) {
6245                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
6246                            rc);
6247                 return rc;
6248         }
6249         if (!BNXT_SINGLE_PF(bp))
6250                 return 0;
6251
6252         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
6253             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
6254             link_info->req_flow_ctrl)
6255                 update_pause = true;
6256         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
6257             link_info->force_pause_setting != link_info->req_flow_ctrl)
6258                 update_pause = true;
6259         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
6260                 if (BNXT_AUTO_MODE(link_info->auto_mode))
6261                         update_link = true;
6262                 if (link_info->req_link_speed != link_info->force_link_speed)
6263                         update_link = true;
6264                 if (link_info->req_duplex != link_info->duplex_setting)
6265                         update_link = true;
6266         } else {
6267                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
6268                         update_link = true;
6269                 if (link_info->advertising != link_info->auto_link_speeds)
6270                         update_link = true;
6271         }
6272
6273         /* The last close may have shutdown the link, so need to call
6274          * PHY_CFG to bring it back up.
6275          */
6276         if (!netif_carrier_ok(bp->dev))
6277                 update_link = true;
6278
6279         if (!bnxt_eee_config_ok(bp))
6280                 update_eee = true;
6281
6282         if (update_link)
6283                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
6284         else if (update_pause)
6285                 rc = bnxt_hwrm_set_pause(bp);
6286         if (rc) {
6287                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
6288                            rc);
6289                 return rc;
6290         }
6291
6292         return rc;
6293 }
6294
6295 /* Common routine to pre-map certain register block to different GRC window.
6296  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
6297  * in PF and 3 windows in VF that can be customized to map in different
6298  * register blocks.
6299  */
6300 static void bnxt_preset_reg_win(struct bnxt *bp)
6301 {
6302         if (BNXT_PF(bp)) {
6303                 /* CAG registers map to GRC window #4 */
6304                 writel(BNXT_CAG_REG_BASE,
6305                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
6306         }
6307 }
6308
6309 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
6310 {
6311         int rc = 0;
6312
6313         bnxt_preset_reg_win(bp);
6314         netif_carrier_off(bp->dev);
6315         if (irq_re_init) {
6316                 rc = bnxt_setup_int_mode(bp);
6317                 if (rc) {
6318                         netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
6319                                    rc);
6320                         return rc;
6321                 }
6322         }
6323         if ((bp->flags & BNXT_FLAG_RFS) &&
6324             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
6325                 /* disable RFS if falling back to INTA */
6326                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
6327                 bp->flags &= ~BNXT_FLAG_RFS;
6328         }
6329
6330         rc = bnxt_alloc_mem(bp, irq_re_init);
6331         if (rc) {
6332                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
6333                 goto open_err_free_mem;
6334         }
6335
6336         if (irq_re_init) {
6337                 bnxt_init_napi(bp);
6338                 rc = bnxt_request_irq(bp);
6339                 if (rc) {
6340                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
6341                         goto open_err;
6342                 }
6343         }
6344
6345         bnxt_enable_napi(bp);
6346
6347         rc = bnxt_init_nic(bp, irq_re_init);
6348         if (rc) {
6349                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
6350                 goto open_err;
6351         }
6352
6353         if (link_re_init) {
6354                 mutex_lock(&bp->link_lock);
6355                 rc = bnxt_update_phy_setting(bp);
6356                 mutex_unlock(&bp->link_lock);
6357                 if (rc)
6358                         netdev_warn(bp->dev, "failed to update phy settings\n");
6359         }
6360
6361         if (irq_re_init)
6362                 udp_tunnel_get_rx_info(bp->dev);
6363
6364         set_bit(BNXT_STATE_OPEN, &bp->state);
6365         bnxt_enable_int(bp);
6366         /* Enable TX queues */
6367         bnxt_tx_enable(bp);
6368         mod_timer(&bp->timer, jiffies + bp->current_interval);
6369         /* Poll link status and check for SFP+ module status */
6370         bnxt_get_port_module_status(bp);
6371
6372         /* VF-reps may need to be re-opened after the PF is re-opened */
6373         if (BNXT_PF(bp))
6374                 bnxt_vf_reps_open(bp);
6375         return 0;
6376
6377 open_err:
6378         bnxt_disable_napi(bp);
6379         bnxt_del_napi(bp);
6380
6381 open_err_free_mem:
6382         bnxt_free_skbs(bp);
6383         bnxt_free_irq(bp);
6384         bnxt_free_mem(bp, true);
6385         return rc;
6386 }
6387
6388 /* rtnl_lock held */
6389 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
6390 {
6391         int rc = 0;
6392
6393         rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
6394         if (rc) {
6395                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
6396                 dev_close(bp->dev);
6397         }
6398         return rc;
6399 }
6400
6401 /* rtnl_lock held, open the NIC half way by allocating all resources, but
6402  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
6403  * self tests.
6404  */
6405 int bnxt_half_open_nic(struct bnxt *bp)
6406 {
6407         int rc = 0;
6408
6409         rc = bnxt_alloc_mem(bp, false);
6410         if (rc) {
6411                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
6412                 goto half_open_err;
6413         }
6414         rc = bnxt_init_nic(bp, false);
6415         if (rc) {
6416                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
6417                 goto half_open_err;
6418         }
6419         return 0;
6420
6421 half_open_err:
6422         bnxt_free_skbs(bp);
6423         bnxt_free_mem(bp, false);
6424         dev_close(bp->dev);
6425         return rc;
6426 }
6427
6428 /* rtnl_lock held, this call can only be made after a previous successful
6429  * call to bnxt_half_open_nic().
6430  */
6431 void bnxt_half_close_nic(struct bnxt *bp)
6432 {
6433         bnxt_hwrm_resource_free(bp, false, false);
6434         bnxt_free_skbs(bp);
6435         bnxt_free_mem(bp, false);
6436 }
6437
6438 static int bnxt_open(struct net_device *dev)
6439 {
6440         struct bnxt *bp = netdev_priv(dev);
6441
6442         return __bnxt_open_nic(bp, true, true);
6443 }
6444
6445 static bool bnxt_drv_busy(struct bnxt *bp)
6446 {
6447         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
6448                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
6449 }
6450
6451 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
6452 {
6453         int rc = 0;
6454
6455 #ifdef CONFIG_BNXT_SRIOV
6456         if (bp->sriov_cfg) {
6457                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
6458                                                       !bp->sriov_cfg,
6459                                                       BNXT_SRIOV_CFG_WAIT_TMO);
6460                 if (rc)
6461                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
6462         }
6463
6464         /* Close the VF-reps before closing PF */
6465         if (BNXT_PF(bp))
6466                 bnxt_vf_reps_close(bp);
6467 #endif
6468         /* Change device state to avoid TX queue wake up's */
6469         bnxt_tx_disable(bp);
6470
6471         clear_bit(BNXT_STATE_OPEN, &bp->state);
6472         smp_mb__after_atomic();
6473         while (bnxt_drv_busy(bp))
6474                 msleep(20);
6475
6476         /* Flush rings and and disable interrupts */
6477         bnxt_shutdown_nic(bp, irq_re_init);
6478
6479         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
6480
6481         bnxt_disable_napi(bp);
6482         del_timer_sync(&bp->timer);
6483         bnxt_free_skbs(bp);
6484
6485         if (irq_re_init) {
6486                 bnxt_free_irq(bp);
6487                 bnxt_del_napi(bp);
6488         }
6489         bnxt_free_mem(bp, irq_re_init);
6490         return rc;
6491 }
6492
6493 static int bnxt_close(struct net_device *dev)
6494 {
6495         struct bnxt *bp = netdev_priv(dev);
6496
6497         bnxt_close_nic(bp, true, true);
6498         bnxt_hwrm_shutdown_link(bp);
6499         return 0;
6500 }
6501
6502 /* rtnl_lock held */
6503 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
6504 {
6505         switch (cmd) {
6506         case SIOCGMIIPHY:
6507                 /* fallthru */
6508         case SIOCGMIIREG: {
6509                 if (!netif_running(dev))
6510                         return -EAGAIN;
6511
6512                 return 0;
6513         }
6514
6515         case SIOCSMIIREG:
6516                 if (!netif_running(dev))
6517                         return -EAGAIN;
6518
6519                 return 0;
6520
6521         default:
6522                 /* do nothing */
6523                 break;
6524         }
6525         return -EOPNOTSUPP;
6526 }
6527
6528 static void
6529 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
6530 {
6531         u32 i;
6532         struct bnxt *bp = netdev_priv(dev);
6533
6534         set_bit(BNXT_STATE_READ_STATS, &bp->state);
6535         /* Make sure bnxt_close_nic() sees that we are reading stats before
6536          * we check the BNXT_STATE_OPEN flag.
6537          */
6538         smp_mb__after_atomic();
6539         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
6540                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
6541                 return;
6542         }
6543
6544         /* TODO check if we need to synchronize with bnxt_close path */
6545         for (i = 0; i < bp->cp_nr_rings; i++) {
6546                 struct bnxt_napi *bnapi = bp->bnapi[i];
6547                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6548                 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
6549
6550                 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
6551                 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
6552                 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
6553
6554                 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
6555                 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
6556                 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
6557
6558                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
6559                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
6560                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
6561
6562                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
6563                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
6564                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
6565
6566                 stats->rx_missed_errors +=
6567                         le64_to_cpu(hw_stats->rx_discard_pkts);
6568
6569                 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
6570
6571                 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
6572         }
6573
6574         if (bp->flags & BNXT_FLAG_PORT_STATS) {
6575                 struct rx_port_stats *rx = bp->hw_rx_port_stats;
6576                 struct tx_port_stats *tx = bp->hw_tx_port_stats;
6577
6578                 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
6579                 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
6580                 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
6581                                           le64_to_cpu(rx->rx_ovrsz_frames) +
6582                                           le64_to_cpu(rx->rx_runt_frames);
6583                 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
6584                                    le64_to_cpu(rx->rx_jbr_frames);
6585                 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
6586                 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
6587                 stats->tx_errors = le64_to_cpu(tx->tx_err);
6588         }
6589         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
6590 }
6591
6592 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
6593 {
6594         struct net_device *dev = bp->dev;
6595         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6596         struct netdev_hw_addr *ha;
6597         u8 *haddr;
6598         int mc_count = 0;
6599         bool update = false;
6600         int off = 0;
6601
6602         netdev_for_each_mc_addr(ha, dev) {
6603                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
6604                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
6605                         vnic->mc_list_count = 0;
6606                         return false;
6607                 }
6608                 haddr = ha->addr;
6609                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
6610                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
6611                         update = true;
6612                 }
6613                 off += ETH_ALEN;
6614                 mc_count++;
6615         }
6616         if (mc_count)
6617                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
6618
6619         if (mc_count != vnic->mc_list_count) {
6620                 vnic->mc_list_count = mc_count;
6621                 update = true;
6622         }
6623         return update;
6624 }
6625
6626 static bool bnxt_uc_list_updated(struct bnxt *bp)
6627 {
6628         struct net_device *dev = bp->dev;
6629         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6630         struct netdev_hw_addr *ha;
6631         int off = 0;
6632
6633         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
6634                 return true;
6635
6636         netdev_for_each_uc_addr(ha, dev) {
6637                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
6638                         return true;
6639
6640                 off += ETH_ALEN;
6641         }
6642         return false;
6643 }
6644
6645 static void bnxt_set_rx_mode(struct net_device *dev)
6646 {
6647         struct bnxt *bp = netdev_priv(dev);
6648         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6649         u32 mask = vnic->rx_mask;
6650         bool mc_update = false;
6651         bool uc_update;
6652
6653         if (!netif_running(dev))
6654                 return;
6655
6656         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
6657                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
6658                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST);
6659
6660         if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
6661                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
6662
6663         uc_update = bnxt_uc_list_updated(bp);
6664
6665         if (dev->flags & IFF_ALLMULTI) {
6666                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
6667                 vnic->mc_list_count = 0;
6668         } else {
6669                 mc_update = bnxt_mc_list_updated(bp, &mask);
6670         }
6671
6672         if (mask != vnic->rx_mask || uc_update || mc_update) {
6673                 vnic->rx_mask = mask;
6674
6675                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
6676                 bnxt_queue_sp_work(bp);
6677         }
6678 }
6679
6680 static int bnxt_cfg_rx_mode(struct bnxt *bp)
6681 {
6682         struct net_device *dev = bp->dev;
6683         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6684         struct netdev_hw_addr *ha;
6685         int i, off = 0, rc;
6686         bool uc_update;
6687
6688         netif_addr_lock_bh(dev);
6689         uc_update = bnxt_uc_list_updated(bp);
6690         netif_addr_unlock_bh(dev);
6691
6692         if (!uc_update)
6693                 goto skip_uc;
6694
6695         mutex_lock(&bp->hwrm_cmd_lock);
6696         for (i = 1; i < vnic->uc_filter_count; i++) {
6697                 struct hwrm_cfa_l2_filter_free_input req = {0};
6698
6699                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
6700                                        -1);
6701
6702                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
6703
6704                 rc = _hwrm_send_message(bp, &req, sizeof(req),
6705                                         HWRM_CMD_TIMEOUT);
6706         }
6707         mutex_unlock(&bp->hwrm_cmd_lock);
6708
6709         vnic->uc_filter_count = 1;
6710
6711         netif_addr_lock_bh(dev);
6712         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
6713                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
6714         } else {
6715                 netdev_for_each_uc_addr(ha, dev) {
6716                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
6717                         off += ETH_ALEN;
6718                         vnic->uc_filter_count++;
6719                 }
6720         }
6721         netif_addr_unlock_bh(dev);
6722
6723         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
6724                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
6725                 if (rc) {
6726                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
6727                                    rc);
6728                         vnic->uc_filter_count = i;
6729                         return rc;
6730                 }
6731         }
6732
6733 skip_uc:
6734         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
6735         if (rc)
6736                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n",
6737                            rc);
6738
6739         return rc;
6740 }
6741
6742 /* If the chip and firmware supports RFS */
6743 static bool bnxt_rfs_supported(struct bnxt *bp)
6744 {
6745         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
6746                 return true;
6747         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
6748                 return true;
6749         return false;
6750 }
6751
6752 /* If runtime conditions support RFS */
6753 static bool bnxt_rfs_capable(struct bnxt *bp)
6754 {
6755 #ifdef CONFIG_RFS_ACCEL
6756         int vnics, max_vnics, max_rss_ctxs;
6757
6758         if (!(bp->flags & BNXT_FLAG_MSIX_CAP))
6759                 return false;
6760
6761         vnics = 1 + bp->rx_nr_rings;
6762         max_vnics = bnxt_get_max_func_vnics(bp);
6763         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
6764
6765         /* RSS contexts not a limiting factor */
6766         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
6767                 max_rss_ctxs = max_vnics;
6768         if (vnics > max_vnics || vnics > max_rss_ctxs) {
6769                 netdev_warn(bp->dev,
6770                             "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
6771                             min(max_rss_ctxs - 1, max_vnics - 1));
6772                 return false;
6773         }
6774
6775         return true;
6776 #else
6777         return false;
6778 #endif
6779 }
6780
6781 static netdev_features_t bnxt_fix_features(struct net_device *dev,
6782                                            netdev_features_t features)
6783 {
6784         struct bnxt *bp = netdev_priv(dev);
6785
6786         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
6787                 features &= ~NETIF_F_NTUPLE;
6788
6789         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
6790          * turned on or off together.
6791          */
6792         if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) !=
6793             (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) {
6794                 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
6795                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
6796                                       NETIF_F_HW_VLAN_STAG_RX);
6797                 else
6798                         features |= NETIF_F_HW_VLAN_CTAG_RX |
6799                                     NETIF_F_HW_VLAN_STAG_RX;
6800         }
6801 #ifdef CONFIG_BNXT_SRIOV
6802         if (BNXT_VF(bp)) {
6803                 if (bp->vf.vlan) {
6804                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
6805                                       NETIF_F_HW_VLAN_STAG_RX);
6806                 }
6807         }
6808 #endif
6809         return features;
6810 }
6811
6812 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
6813 {
6814         struct bnxt *bp = netdev_priv(dev);
6815         u32 flags = bp->flags;
6816         u32 changes;
6817         int rc = 0;
6818         bool re_init = false;
6819         bool update_tpa = false;
6820
6821         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
6822         if ((features & NETIF_F_GRO) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
6823                 flags |= BNXT_FLAG_GRO;
6824         if (features & NETIF_F_LRO)
6825                 flags |= BNXT_FLAG_LRO;
6826
6827         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
6828                 flags &= ~BNXT_FLAG_TPA;
6829
6830         if (features & NETIF_F_HW_VLAN_CTAG_RX)
6831                 flags |= BNXT_FLAG_STRIP_VLAN;
6832
6833         if (features & NETIF_F_NTUPLE)
6834                 flags |= BNXT_FLAG_RFS;
6835
6836         changes = flags ^ bp->flags;
6837         if (changes & BNXT_FLAG_TPA) {
6838                 update_tpa = true;
6839                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
6840                     (flags & BNXT_FLAG_TPA) == 0)
6841                         re_init = true;
6842         }
6843
6844         if (changes & ~BNXT_FLAG_TPA)
6845                 re_init = true;
6846
6847         if (flags != bp->flags) {
6848                 u32 old_flags = bp->flags;
6849
6850                 bp->flags = flags;
6851
6852                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
6853                         if (update_tpa)
6854                                 bnxt_set_ring_params(bp);
6855                         return rc;
6856                 }
6857
6858                 if (re_init) {
6859                         bnxt_close_nic(bp, false, false);
6860                         if (update_tpa)
6861                                 bnxt_set_ring_params(bp);
6862
6863                         return bnxt_open_nic(bp, false, false);
6864                 }
6865                 if (update_tpa) {
6866                         rc = bnxt_set_tpa(bp,
6867                                           (flags & BNXT_FLAG_TPA) ?
6868                                           true : false);
6869                         if (rc)
6870                                 bp->flags = old_flags;
6871                 }
6872         }
6873         return rc;
6874 }
6875
6876 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
6877 {
6878         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
6879         int i = bnapi->index;
6880
6881         if (!txr)
6882                 return;
6883
6884         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
6885                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
6886                     txr->tx_cons);
6887 }
6888
6889 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
6890 {
6891         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
6892         int i = bnapi->index;
6893
6894         if (!rxr)
6895                 return;
6896
6897         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
6898                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
6899                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
6900                     rxr->rx_sw_agg_prod);
6901 }
6902
6903 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
6904 {
6905         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6906         int i = bnapi->index;
6907
6908         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
6909                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
6910 }
6911
6912 static void bnxt_dbg_dump_states(struct bnxt *bp)
6913 {
6914         int i;
6915         struct bnxt_napi *bnapi;
6916
6917         for (i = 0; i < bp->cp_nr_rings; i++) {
6918                 bnapi = bp->bnapi[i];
6919                 if (netif_msg_drv(bp)) {
6920                         bnxt_dump_tx_sw_state(bnapi);
6921                         bnxt_dump_rx_sw_state(bnapi);
6922                         bnxt_dump_cp_sw_state(bnapi);
6923                 }
6924         }
6925 }
6926
6927 static void bnxt_reset_task(struct bnxt *bp, bool silent)
6928 {
6929         if (!silent)
6930                 bnxt_dbg_dump_states(bp);
6931         if (netif_running(bp->dev)) {
6932                 int rc;
6933
6934                 if (!silent)
6935                         bnxt_ulp_stop(bp);
6936                 bnxt_close_nic(bp, false, false);
6937                 rc = bnxt_open_nic(bp, false, false);
6938                 if (!silent && !rc)
6939                         bnxt_ulp_start(bp);
6940         }
6941 }
6942
6943 static void bnxt_tx_timeout(struct net_device *dev)
6944 {
6945         struct bnxt *bp = netdev_priv(dev);
6946
6947         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
6948         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
6949         bnxt_queue_sp_work(bp);
6950 }
6951
6952 #ifdef CONFIG_NET_POLL_CONTROLLER
6953 static void bnxt_poll_controller(struct net_device *dev)
6954 {
6955         struct bnxt *bp = netdev_priv(dev);
6956         int i;
6957
6958         /* Only process tx rings/combined rings in netpoll mode. */
6959         for (i = 0; i < bp->tx_nr_rings; i++) {
6960                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
6961
6962                 napi_schedule(&txr->bnapi->napi);
6963         }
6964 }
6965 #endif
6966
6967 static void bnxt_timer(unsigned long data)
6968 {
6969         struct bnxt *bp = (struct bnxt *)data;
6970         struct net_device *dev = bp->dev;
6971
6972         if (!netif_running(dev))
6973                 return;
6974
6975         if (atomic_read(&bp->intr_sem) != 0)
6976                 goto bnxt_restart_timer;
6977
6978         if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) &&
6979             bp->stats_coal_ticks) {
6980                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
6981                 bnxt_queue_sp_work(bp);
6982         }
6983 bnxt_restart_timer:
6984         mod_timer(&bp->timer, jiffies + bp->current_interval);
6985 }
6986
6987 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
6988 {
6989         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
6990          * set.  If the device is being closed, bnxt_close() may be holding
6991          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
6992          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
6993          */
6994         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
6995         rtnl_lock();
6996 }
6997
6998 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
6999 {
7000         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7001         rtnl_unlock();
7002 }
7003
7004 /* Only called from bnxt_sp_task() */
7005 static void bnxt_reset(struct bnxt *bp, bool silent)
7006 {
7007         bnxt_rtnl_lock_sp(bp);
7008         if (test_bit(BNXT_STATE_OPEN, &bp->state))
7009                 bnxt_reset_task(bp, silent);
7010         bnxt_rtnl_unlock_sp(bp);
7011 }
7012
7013 static void bnxt_cfg_ntp_filters(struct bnxt *);
7014
7015 static void bnxt_sp_task(struct work_struct *work)
7016 {
7017         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
7018
7019         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7020         smp_mb__after_atomic();
7021         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
7022                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7023                 return;
7024         }
7025
7026         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
7027                 bnxt_cfg_rx_mode(bp);
7028
7029         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
7030                 bnxt_cfg_ntp_filters(bp);
7031         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
7032                 bnxt_hwrm_exec_fwd_req(bp);
7033         if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
7034                 bnxt_hwrm_tunnel_dst_port_alloc(
7035                         bp, bp->vxlan_port,
7036                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7037         }
7038         if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
7039                 bnxt_hwrm_tunnel_dst_port_free(
7040                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7041         }
7042         if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) {
7043                 bnxt_hwrm_tunnel_dst_port_alloc(
7044                         bp, bp->nge_port,
7045                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7046         }
7047         if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) {
7048                 bnxt_hwrm_tunnel_dst_port_free(
7049                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7050         }
7051         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event))
7052                 bnxt_hwrm_port_qstats(bp);
7053
7054         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
7055                 int rc;
7056
7057                 mutex_lock(&bp->link_lock);
7058                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
7059                                        &bp->sp_event))
7060                         bnxt_hwrm_phy_qcaps(bp);
7061
7062                 rc = bnxt_update_link(bp, true);
7063                 mutex_unlock(&bp->link_lock);
7064                 if (rc)
7065                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
7066                                    rc);
7067         }
7068         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
7069                 mutex_lock(&bp->link_lock);
7070                 bnxt_get_port_module_status(bp);
7071                 mutex_unlock(&bp->link_lock);
7072         }
7073         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
7074          * must be the last functions to be called before exiting.
7075          */
7076         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
7077                 bnxt_reset(bp, false);
7078
7079         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
7080                 bnxt_reset(bp, true);
7081
7082         smp_mb__before_atomic();
7083         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7084 }
7085
7086 /* Under rtnl_lock */
7087 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
7088                      int tx_xdp)
7089 {
7090         int max_rx, max_tx, tx_sets = 1;
7091         int tx_rings_needed;
7092         int rc;
7093
7094         if (tcs)
7095                 tx_sets = tcs;
7096
7097         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
7098         if (rc)
7099                 return rc;
7100
7101         if (max_rx < rx)
7102                 return -ENOMEM;
7103
7104         tx_rings_needed = tx * tx_sets + tx_xdp;
7105         if (max_tx < tx_rings_needed)
7106                 return -ENOMEM;
7107
7108         return bnxt_hwrm_check_tx_rings(bp, tx_rings_needed);
7109 }
7110
7111 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
7112 {
7113         if (bp->bar2) {
7114                 pci_iounmap(pdev, bp->bar2);
7115                 bp->bar2 = NULL;
7116         }
7117
7118         if (bp->bar1) {
7119                 pci_iounmap(pdev, bp->bar1);
7120                 bp->bar1 = NULL;
7121         }
7122
7123         if (bp->bar0) {
7124                 pci_iounmap(pdev, bp->bar0);
7125                 bp->bar0 = NULL;
7126         }
7127 }
7128
7129 static void bnxt_cleanup_pci(struct bnxt *bp)
7130 {
7131         bnxt_unmap_bars(bp, bp->pdev);
7132         pci_release_regions(bp->pdev);
7133         pci_disable_device(bp->pdev);
7134 }
7135
7136 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
7137 {
7138         int rc;
7139         struct bnxt *bp = netdev_priv(dev);
7140
7141         SET_NETDEV_DEV(dev, &pdev->dev);
7142
7143         /* enable device (incl. PCI PM wakeup), and bus-mastering */
7144         rc = pci_enable_device(pdev);
7145         if (rc) {
7146                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
7147                 goto init_err;
7148         }
7149
7150         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
7151                 dev_err(&pdev->dev,
7152                         "Cannot find PCI device base address, aborting\n");
7153                 rc = -ENODEV;
7154                 goto init_err_disable;
7155         }
7156
7157         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
7158         if (rc) {
7159                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
7160                 goto init_err_disable;
7161         }
7162
7163         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
7164             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
7165                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
7166                 goto init_err_disable;
7167         }
7168
7169         pci_set_master(pdev);
7170
7171         bp->dev = dev;
7172         bp->pdev = pdev;
7173
7174         bp->bar0 = pci_ioremap_bar(pdev, 0);
7175         if (!bp->bar0) {
7176                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
7177                 rc = -ENOMEM;
7178                 goto init_err_release;
7179         }
7180
7181         bp->bar1 = pci_ioremap_bar(pdev, 2);
7182         if (!bp->bar1) {
7183                 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
7184                 rc = -ENOMEM;
7185                 goto init_err_release;
7186         }
7187
7188         bp->bar2 = pci_ioremap_bar(pdev, 4);
7189         if (!bp->bar2) {
7190                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
7191                 rc = -ENOMEM;
7192                 goto init_err_release;
7193         }
7194
7195         pci_enable_pcie_error_reporting(pdev);
7196
7197         INIT_WORK(&bp->sp_task, bnxt_sp_task);
7198
7199         spin_lock_init(&bp->ntp_fltr_lock);
7200
7201         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
7202         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
7203
7204         /* tick values in micro seconds */
7205         bp->rx_coal_ticks = 12;
7206         bp->rx_coal_bufs = 30;
7207         bp->rx_coal_ticks_irq = 1;
7208         bp->rx_coal_bufs_irq = 2;
7209
7210         bp->tx_coal_ticks = 25;
7211         bp->tx_coal_bufs = 30;
7212         bp->tx_coal_ticks_irq = 2;
7213         bp->tx_coal_bufs_irq = 2;
7214
7215         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
7216
7217         init_timer(&bp->timer);
7218         bp->timer.data = (unsigned long)bp;
7219         bp->timer.function = bnxt_timer;
7220         bp->current_interval = BNXT_TIMER_INTERVAL;
7221
7222         clear_bit(BNXT_STATE_OPEN, &bp->state);
7223         return 0;
7224
7225 init_err_release:
7226         bnxt_unmap_bars(bp, pdev);
7227         pci_release_regions(pdev);
7228
7229 init_err_disable:
7230         pci_disable_device(pdev);
7231
7232 init_err:
7233         return rc;
7234 }
7235
7236 /* rtnl_lock held */
7237 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
7238 {
7239         struct sockaddr *addr = p;
7240         struct bnxt *bp = netdev_priv(dev);
7241         int rc = 0;
7242
7243         if (!is_valid_ether_addr(addr->sa_data))
7244                 return -EADDRNOTAVAIL;
7245
7246         rc = bnxt_approve_mac(bp, addr->sa_data);
7247         if (rc)
7248                 return rc;
7249
7250         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
7251                 return 0;
7252
7253         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
7254         if (netif_running(dev)) {
7255                 bnxt_close_nic(bp, false, false);
7256                 rc = bnxt_open_nic(bp, false, false);
7257         }
7258
7259         return rc;
7260 }
7261
7262 /* rtnl_lock held */
7263 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
7264 {
7265         struct bnxt *bp = netdev_priv(dev);
7266
7267         if (netif_running(dev))
7268                 bnxt_close_nic(bp, false, false);
7269
7270         dev->mtu = new_mtu;
7271         bnxt_set_ring_params(bp);
7272
7273         if (netif_running(dev))
7274                 return bnxt_open_nic(bp, false, false);
7275
7276         return 0;
7277 }
7278
7279 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
7280 {
7281         struct bnxt *bp = netdev_priv(dev);
7282         bool sh = false;
7283         int rc;
7284
7285         if (tc > bp->max_tc) {
7286                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
7287                            tc, bp->max_tc);
7288                 return -EINVAL;
7289         }
7290
7291         if (netdev_get_num_tc(dev) == tc)
7292                 return 0;
7293
7294         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7295                 sh = true;
7296
7297         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
7298                               sh, tc, bp->tx_nr_rings_xdp);
7299         if (rc)
7300                 return rc;
7301
7302         /* Needs to close the device and do hw resource re-allocations */
7303         if (netif_running(bp->dev))
7304                 bnxt_close_nic(bp, true, false);
7305
7306         if (tc) {
7307                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
7308                 netdev_set_num_tc(dev, tc);
7309         } else {
7310                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
7311                 netdev_reset_tc(dev);
7312         }
7313         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
7314         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
7315                                bp->tx_nr_rings + bp->rx_nr_rings;
7316         bp->num_stat_ctxs = bp->cp_nr_rings;
7317
7318         if (netif_running(bp->dev))
7319                 return bnxt_open_nic(bp, true, false);
7320
7321         return 0;
7322 }
7323
7324 static int bnxt_setup_flower(struct net_device *dev,
7325                              struct tc_cls_flower_offload *cls_flower)
7326 {
7327         struct bnxt *bp = netdev_priv(dev);
7328
7329         if (BNXT_VF(bp))
7330                 return -EOPNOTSUPP;
7331
7332         return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, cls_flower);
7333 }
7334
7335 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
7336                          void *type_data)
7337 {
7338         switch (type) {
7339         case TC_SETUP_CLSFLOWER:
7340                 return bnxt_setup_flower(dev, type_data);
7341         case TC_SETUP_MQPRIO: {
7342                 struct tc_mqprio_qopt *mqprio = type_data;
7343
7344                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
7345
7346                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
7347         }
7348         default:
7349                 return -EOPNOTSUPP;
7350         }
7351 }
7352
7353 #ifdef CONFIG_RFS_ACCEL
7354 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
7355                             struct bnxt_ntuple_filter *f2)
7356 {
7357         struct flow_keys *keys1 = &f1->fkeys;
7358         struct flow_keys *keys2 = &f2->fkeys;
7359
7360         if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
7361             keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
7362             keys1->ports.ports == keys2->ports.ports &&
7363             keys1->basic.ip_proto == keys2->basic.ip_proto &&
7364             keys1->basic.n_proto == keys2->basic.n_proto &&
7365             keys1->control.flags == keys2->control.flags &&
7366             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
7367             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
7368                 return true;
7369
7370         return false;
7371 }
7372
7373 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
7374                               u16 rxq_index, u32 flow_id)
7375 {
7376         struct bnxt *bp = netdev_priv(dev);
7377         struct bnxt_ntuple_filter *fltr, *new_fltr;
7378         struct flow_keys *fkeys;
7379         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
7380         int rc = 0, idx, bit_id, l2_idx = 0;
7381         struct hlist_head *head;
7382
7383         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
7384                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
7385                 int off = 0, j;
7386
7387                 netif_addr_lock_bh(dev);
7388                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
7389                         if (ether_addr_equal(eth->h_dest,
7390                                              vnic->uc_list + off)) {
7391                                 l2_idx = j + 1;
7392                                 break;
7393                         }
7394                 }
7395                 netif_addr_unlock_bh(dev);
7396                 if (!l2_idx)
7397                         return -EINVAL;
7398         }
7399         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
7400         if (!new_fltr)
7401                 return -ENOMEM;
7402
7403         fkeys = &new_fltr->fkeys;
7404         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
7405                 rc = -EPROTONOSUPPORT;
7406                 goto err_free;
7407         }
7408
7409         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
7410              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
7411             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
7412              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
7413                 rc = -EPROTONOSUPPORT;
7414                 goto err_free;
7415         }
7416         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
7417             bp->hwrm_spec_code < 0x10601) {
7418                 rc = -EPROTONOSUPPORT;
7419                 goto err_free;
7420         }
7421         if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) &&
7422             bp->hwrm_spec_code < 0x10601) {
7423                 rc = -EPROTONOSUPPORT;
7424                 goto err_free;
7425         }
7426
7427         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
7428         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
7429
7430         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
7431         head = &bp->ntp_fltr_hash_tbl[idx];
7432         rcu_read_lock();
7433         hlist_for_each_entry_rcu(fltr, head, hash) {
7434                 if (bnxt_fltr_match(fltr, new_fltr)) {
7435                         rcu_read_unlock();
7436                         rc = 0;
7437                         goto err_free;
7438                 }
7439         }
7440         rcu_read_unlock();
7441
7442         spin_lock_bh(&bp->ntp_fltr_lock);
7443         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
7444                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
7445         if (bit_id < 0) {
7446                 spin_unlock_bh(&bp->ntp_fltr_lock);
7447                 rc = -ENOMEM;
7448                 goto err_free;
7449         }
7450
7451         new_fltr->sw_id = (u16)bit_id;
7452         new_fltr->flow_id = flow_id;
7453         new_fltr->l2_fltr_idx = l2_idx;
7454         new_fltr->rxq = rxq_index;
7455         hlist_add_head_rcu(&new_fltr->hash, head);
7456         bp->ntp_fltr_count++;
7457         spin_unlock_bh(&bp->ntp_fltr_lock);
7458
7459         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
7460         bnxt_queue_sp_work(bp);
7461
7462         return new_fltr->sw_id;
7463
7464 err_free:
7465         kfree(new_fltr);
7466         return rc;
7467 }
7468
7469 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
7470 {
7471         int i;
7472
7473         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
7474                 struct hlist_head *head;
7475                 struct hlist_node *tmp;
7476                 struct bnxt_ntuple_filter *fltr;
7477                 int rc;
7478
7479                 head = &bp->ntp_fltr_hash_tbl[i];
7480                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
7481                         bool del = false;
7482
7483                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
7484                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
7485                                                         fltr->flow_id,
7486                                                         fltr->sw_id)) {
7487                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
7488                                                                          fltr);
7489                                         del = true;
7490                                 }
7491                         } else {
7492                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
7493                                                                        fltr);
7494                                 if (rc)
7495                                         del = true;
7496                                 else
7497                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
7498                         }
7499
7500                         if (del) {
7501                                 spin_lock_bh(&bp->ntp_fltr_lock);
7502                                 hlist_del_rcu(&fltr->hash);
7503                                 bp->ntp_fltr_count--;
7504                                 spin_unlock_bh(&bp->ntp_fltr_lock);
7505                                 synchronize_rcu();
7506                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
7507                                 kfree(fltr);
7508                         }
7509                 }
7510         }
7511         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
7512                 netdev_info(bp->dev, "Receive PF driver unload event!");
7513 }
7514
7515 #else
7516
7517 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
7518 {
7519 }
7520
7521 #endif /* CONFIG_RFS_ACCEL */
7522
7523 static void bnxt_udp_tunnel_add(struct net_device *dev,
7524                                 struct udp_tunnel_info *ti)
7525 {
7526         struct bnxt *bp = netdev_priv(dev);
7527
7528         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
7529                 return;
7530
7531         if (!netif_running(dev))
7532                 return;
7533
7534         switch (ti->type) {
7535         case UDP_TUNNEL_TYPE_VXLAN:
7536                 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port)
7537                         return;
7538
7539                 bp->vxlan_port_cnt++;
7540                 if (bp->vxlan_port_cnt == 1) {
7541                         bp->vxlan_port = ti->port;
7542                         set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
7543                         bnxt_queue_sp_work(bp);
7544                 }
7545                 break;
7546         case UDP_TUNNEL_TYPE_GENEVE:
7547                 if (bp->nge_port_cnt && bp->nge_port != ti->port)
7548                         return;
7549
7550                 bp->nge_port_cnt++;
7551                 if (bp->nge_port_cnt == 1) {
7552                         bp->nge_port = ti->port;
7553                         set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event);
7554                 }
7555                 break;
7556         default:
7557                 return;
7558         }
7559
7560         bnxt_queue_sp_work(bp);
7561 }
7562
7563 static void bnxt_udp_tunnel_del(struct net_device *dev,
7564                                 struct udp_tunnel_info *ti)
7565 {
7566         struct bnxt *bp = netdev_priv(dev);
7567
7568         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
7569                 return;
7570
7571         if (!netif_running(dev))
7572                 return;
7573
7574         switch (ti->type) {
7575         case UDP_TUNNEL_TYPE_VXLAN:
7576                 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port)
7577                         return;
7578                 bp->vxlan_port_cnt--;
7579
7580                 if (bp->vxlan_port_cnt != 0)
7581                         return;
7582
7583                 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
7584                 break;
7585         case UDP_TUNNEL_TYPE_GENEVE:
7586                 if (!bp->nge_port_cnt || bp->nge_port != ti->port)
7587                         return;
7588                 bp->nge_port_cnt--;
7589
7590                 if (bp->nge_port_cnt != 0)
7591                         return;
7592
7593                 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event);
7594                 break;
7595         default:
7596                 return;
7597         }
7598
7599         bnxt_queue_sp_work(bp);
7600 }
7601
7602 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
7603                                struct net_device *dev, u32 filter_mask,
7604                                int nlflags)
7605 {
7606         struct bnxt *bp = netdev_priv(dev);
7607
7608         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
7609                                        nlflags, filter_mask, NULL);
7610 }
7611
7612 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
7613                                u16 flags)
7614 {
7615         struct bnxt *bp = netdev_priv(dev);
7616         struct nlattr *attr, *br_spec;
7617         int rem, rc = 0;
7618
7619         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
7620                 return -EOPNOTSUPP;
7621
7622         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
7623         if (!br_spec)
7624                 return -EINVAL;
7625
7626         nla_for_each_nested(attr, br_spec, rem) {
7627                 u16 mode;
7628
7629                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
7630                         continue;
7631
7632                 if (nla_len(attr) < sizeof(mode))
7633                         return -EINVAL;
7634
7635                 mode = nla_get_u16(attr);
7636                 if (mode == bp->br_mode)
7637                         break;
7638
7639                 rc = bnxt_hwrm_set_br_mode(bp, mode);
7640                 if (!rc)
7641                         bp->br_mode = mode;
7642                 break;
7643         }
7644         return rc;
7645 }
7646
7647 static int bnxt_get_phys_port_name(struct net_device *dev, char *buf,
7648                                    size_t len)
7649 {
7650         struct bnxt *bp = netdev_priv(dev);
7651         int rc;
7652
7653         /* The PF and it's VF-reps only support the switchdev framework */
7654         if (!BNXT_PF(bp))
7655                 return -EOPNOTSUPP;
7656
7657         rc = snprintf(buf, len, "p%d", bp->pf.port_id);
7658
7659         if (rc >= len)
7660                 return -EOPNOTSUPP;
7661         return 0;
7662 }
7663
7664 int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr)
7665 {
7666         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
7667                 return -EOPNOTSUPP;
7668
7669         /* The PF and it's VF-reps only support the switchdev framework */
7670         if (!BNXT_PF(bp))
7671                 return -EOPNOTSUPP;
7672
7673         switch (attr->id) {
7674         case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
7675                 /* In SRIOV each PF-pool (PF + child VFs) serves as a
7676                  * switching domain, the PF's perm mac-addr can be used
7677                  * as the unique parent-id
7678                  */
7679                 attr->u.ppid.id_len = ETH_ALEN;
7680                 ether_addr_copy(attr->u.ppid.id, bp->pf.mac_addr);
7681                 break;
7682         default:
7683                 return -EOPNOTSUPP;
7684         }
7685         return 0;
7686 }
7687
7688 static int bnxt_swdev_port_attr_get(struct net_device *dev,
7689                                     struct switchdev_attr *attr)
7690 {
7691         return bnxt_port_attr_get(netdev_priv(dev), attr);
7692 }
7693
7694 static const struct switchdev_ops bnxt_switchdev_ops = {
7695         .switchdev_port_attr_get        = bnxt_swdev_port_attr_get
7696 };
7697
7698 static const struct net_device_ops bnxt_netdev_ops = {
7699         .ndo_open               = bnxt_open,
7700         .ndo_start_xmit         = bnxt_start_xmit,
7701         .ndo_stop               = bnxt_close,
7702         .ndo_get_stats64        = bnxt_get_stats64,
7703         .ndo_set_rx_mode        = bnxt_set_rx_mode,
7704         .ndo_do_ioctl           = bnxt_ioctl,
7705         .ndo_validate_addr      = eth_validate_addr,
7706         .ndo_set_mac_address    = bnxt_change_mac_addr,
7707         .ndo_change_mtu         = bnxt_change_mtu,
7708         .ndo_fix_features       = bnxt_fix_features,
7709         .ndo_set_features       = bnxt_set_features,
7710         .ndo_tx_timeout         = bnxt_tx_timeout,
7711 #ifdef CONFIG_BNXT_SRIOV
7712         .ndo_get_vf_config      = bnxt_get_vf_config,
7713         .ndo_set_vf_mac         = bnxt_set_vf_mac,
7714         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
7715         .ndo_set_vf_rate        = bnxt_set_vf_bw,
7716         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
7717         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
7718 #endif
7719 #ifdef CONFIG_NET_POLL_CONTROLLER
7720         .ndo_poll_controller    = bnxt_poll_controller,
7721 #endif
7722         .ndo_setup_tc           = bnxt_setup_tc,
7723 #ifdef CONFIG_RFS_ACCEL
7724         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
7725 #endif
7726         .ndo_udp_tunnel_add     = bnxt_udp_tunnel_add,
7727         .ndo_udp_tunnel_del     = bnxt_udp_tunnel_del,
7728         .ndo_xdp                = bnxt_xdp,
7729         .ndo_bridge_getlink     = bnxt_bridge_getlink,
7730         .ndo_bridge_setlink     = bnxt_bridge_setlink,
7731         .ndo_get_phys_port_name = bnxt_get_phys_port_name
7732 };
7733
7734 static void bnxt_remove_one(struct pci_dev *pdev)
7735 {
7736         struct net_device *dev = pci_get_drvdata(pdev);
7737         struct bnxt *bp = netdev_priv(dev);
7738
7739         if (BNXT_PF(bp)) {
7740                 bnxt_sriov_disable(bp);
7741                 bnxt_dl_unregister(bp);
7742         }
7743
7744         pci_disable_pcie_error_reporting(pdev);
7745         unregister_netdev(dev);
7746         bnxt_shutdown_tc(bp);
7747         bnxt_cancel_sp_work(bp);
7748         bp->sp_event = 0;
7749
7750         bnxt_clear_int_mode(bp);
7751         bnxt_hwrm_func_drv_unrgtr(bp);
7752         bnxt_free_hwrm_resources(bp);
7753         bnxt_free_hwrm_short_cmd_req(bp);
7754         bnxt_ethtool_free(bp);
7755         bnxt_dcb_free(bp);
7756         kfree(bp->edev);
7757         bp->edev = NULL;
7758         if (bp->xdp_prog)
7759                 bpf_prog_put(bp->xdp_prog);
7760         bnxt_cleanup_pci(bp);
7761         free_netdev(dev);
7762 }
7763
7764 static int bnxt_probe_phy(struct bnxt *bp)
7765 {
7766         int rc = 0;
7767         struct bnxt_link_info *link_info = &bp->link_info;
7768
7769         rc = bnxt_hwrm_phy_qcaps(bp);
7770         if (rc) {
7771                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
7772                            rc);
7773                 return rc;
7774         }
7775         mutex_init(&bp->link_lock);
7776
7777         rc = bnxt_update_link(bp, false);
7778         if (rc) {
7779                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
7780                            rc);
7781                 return rc;
7782         }
7783
7784         /* Older firmware does not have supported_auto_speeds, so assume
7785          * that all supported speeds can be autonegotiated.
7786          */
7787         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
7788                 link_info->support_auto_speeds = link_info->support_speeds;
7789
7790         /*initialize the ethool setting copy with NVM settings */
7791         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
7792                 link_info->autoneg = BNXT_AUTONEG_SPEED;
7793                 if (bp->hwrm_spec_code >= 0x10201) {
7794                         if (link_info->auto_pause_setting &
7795                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
7796                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
7797                 } else {
7798                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
7799                 }
7800                 link_info->advertising = link_info->auto_link_speeds;
7801         } else {
7802                 link_info->req_link_speed = link_info->force_link_speed;
7803                 link_info->req_duplex = link_info->duplex_setting;
7804         }
7805         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
7806                 link_info->req_flow_ctrl =
7807                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
7808         else
7809                 link_info->req_flow_ctrl = link_info->force_pause_setting;
7810         return rc;
7811 }
7812
7813 static int bnxt_get_max_irq(struct pci_dev *pdev)
7814 {
7815         u16 ctrl;
7816
7817         if (!pdev->msix_cap)
7818                 return 1;
7819
7820         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
7821         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
7822 }
7823
7824 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
7825                                 int *max_cp)
7826 {
7827         int max_ring_grps = 0;
7828
7829 #ifdef CONFIG_BNXT_SRIOV
7830         if (!BNXT_PF(bp)) {
7831                 *max_tx = bp->vf.max_tx_rings;
7832                 *max_rx = bp->vf.max_rx_rings;
7833                 *max_cp = min_t(int, bp->vf.max_irqs, bp->vf.max_cp_rings);
7834                 *max_cp = min_t(int, *max_cp, bp->vf.max_stat_ctxs);
7835                 max_ring_grps = bp->vf.max_hw_ring_grps;
7836         } else
7837 #endif
7838         {
7839                 *max_tx = bp->pf.max_tx_rings;
7840                 *max_rx = bp->pf.max_rx_rings;
7841                 *max_cp = min_t(int, bp->pf.max_irqs, bp->pf.max_cp_rings);
7842                 *max_cp = min_t(int, *max_cp, bp->pf.max_stat_ctxs);
7843                 max_ring_grps = bp->pf.max_hw_ring_grps;
7844         }
7845         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
7846                 *max_cp -= 1;
7847                 *max_rx -= 2;
7848         }
7849         if (bp->flags & BNXT_FLAG_AGG_RINGS)
7850                 *max_rx >>= 1;
7851         *max_rx = min_t(int, *max_rx, max_ring_grps);
7852 }
7853
7854 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
7855 {
7856         int rx, tx, cp;
7857
7858         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
7859         if (!rx || !tx || !cp)
7860                 return -ENOMEM;
7861
7862         *max_rx = rx;
7863         *max_tx = tx;
7864         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
7865 }
7866
7867 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
7868                                bool shared)
7869 {
7870         int rc;
7871
7872         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
7873         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
7874                 /* Not enough rings, try disabling agg rings. */
7875                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
7876                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
7877                 if (rc)
7878                         return rc;
7879                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
7880                 bp->dev->hw_features &= ~NETIF_F_LRO;
7881                 bp->dev->features &= ~NETIF_F_LRO;
7882                 bnxt_set_ring_params(bp);
7883         }
7884
7885         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
7886                 int max_cp, max_stat, max_irq;
7887
7888                 /* Reserve minimum resources for RoCE */
7889                 max_cp = bnxt_get_max_func_cp_rings(bp);
7890                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
7891                 max_irq = bnxt_get_max_func_irqs(bp);
7892                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
7893                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
7894                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
7895                         return 0;
7896
7897                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
7898                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
7899                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
7900                 max_cp = min_t(int, max_cp, max_irq);
7901                 max_cp = min_t(int, max_cp, max_stat);
7902                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
7903                 if (rc)
7904                         rc = 0;
7905         }
7906         return rc;
7907 }
7908
7909 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
7910 {
7911         int dflt_rings, max_rx_rings, max_tx_rings, rc;
7912
7913         if (sh)
7914                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
7915         dflt_rings = netif_get_num_default_rss_queues();
7916         /* Reduce default rings to reduce memory usage on multi-port cards */
7917         if (bp->port_count > 1)
7918                 dflt_rings = min_t(int, dflt_rings, 4);
7919         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
7920         if (rc)
7921                 return rc;
7922         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
7923         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
7924
7925         rc = bnxt_hwrm_reserve_tx_rings(bp, &bp->tx_nr_rings_per_tc);
7926         if (rc)
7927                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
7928
7929         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
7930         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
7931                                bp->tx_nr_rings + bp->rx_nr_rings;
7932         bp->num_stat_ctxs = bp->cp_nr_rings;
7933         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7934                 bp->rx_nr_rings++;
7935                 bp->cp_nr_rings++;
7936         }
7937         return rc;
7938 }
7939
7940 void bnxt_restore_pf_fw_resources(struct bnxt *bp)
7941 {
7942         ASSERT_RTNL();
7943         bnxt_hwrm_func_qcaps(bp);
7944         bnxt_subtract_ulp_resources(bp, BNXT_ROCE_ULP);
7945 }
7946
7947 static int bnxt_init_mac_addr(struct bnxt *bp)
7948 {
7949         int rc = 0;
7950
7951         if (BNXT_PF(bp)) {
7952                 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
7953         } else {
7954 #ifdef CONFIG_BNXT_SRIOV
7955                 struct bnxt_vf_info *vf = &bp->vf;
7956
7957                 if (is_valid_ether_addr(vf->mac_addr)) {
7958                         /* overwrite netdev dev_adr with admin VF MAC */
7959                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
7960                 } else {
7961                         eth_hw_addr_random(bp->dev);
7962                         rc = bnxt_approve_mac(bp, bp->dev->dev_addr);
7963                 }
7964 #endif
7965         }
7966         return rc;
7967 }
7968
7969 static void bnxt_parse_log_pcie_link(struct bnxt *bp)
7970 {
7971         enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN;
7972         enum pci_bus_speed speed = PCI_SPEED_UNKNOWN;
7973
7974         if (pcie_get_minimum_link(pci_physfn(bp->pdev), &speed, &width) ||
7975             speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
7976                 netdev_info(bp->dev, "Failed to determine PCIe Link Info\n");
7977         else
7978                 netdev_info(bp->dev, "PCIe: Speed %s Width x%d\n",
7979                             speed == PCIE_SPEED_2_5GT ? "2.5GT/s" :
7980                             speed == PCIE_SPEED_5_0GT ? "5.0GT/s" :
7981                             speed == PCIE_SPEED_8_0GT ? "8.0GT/s" :
7982                             "Unknown", width);
7983 }
7984
7985 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
7986 {
7987         static int version_printed;
7988         struct net_device *dev;
7989         struct bnxt *bp;
7990         int rc, max_irqs;
7991
7992         if (pci_is_bridge(pdev))
7993                 return -ENODEV;
7994
7995         if (version_printed++ == 0)
7996                 pr_info("%s", version);
7997
7998         max_irqs = bnxt_get_max_irq(pdev);
7999         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
8000         if (!dev)
8001                 return -ENOMEM;
8002
8003         bp = netdev_priv(dev);
8004
8005         if (bnxt_vf_pciid(ent->driver_data))
8006                 bp->flags |= BNXT_FLAG_VF;
8007
8008         if (pdev->msix_cap)
8009                 bp->flags |= BNXT_FLAG_MSIX_CAP;
8010
8011         rc = bnxt_init_board(pdev, dev);
8012         if (rc < 0)
8013                 goto init_err_free;
8014
8015         dev->netdev_ops = &bnxt_netdev_ops;
8016         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
8017         dev->ethtool_ops = &bnxt_ethtool_ops;
8018         SWITCHDEV_SET_OPS(dev, &bnxt_switchdev_ops);
8019         pci_set_drvdata(pdev, dev);
8020
8021         rc = bnxt_alloc_hwrm_resources(bp);
8022         if (rc)
8023                 goto init_err_pci_clean;
8024
8025         mutex_init(&bp->hwrm_cmd_lock);
8026         rc = bnxt_hwrm_ver_get(bp);
8027         if (rc)
8028                 goto init_err_pci_clean;
8029
8030         if (bp->flags & BNXT_FLAG_SHORT_CMD) {
8031                 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
8032                 if (rc)
8033                         goto init_err_pci_clean;
8034         }
8035
8036         rc = bnxt_hwrm_func_reset(bp);
8037         if (rc)
8038                 goto init_err_pci_clean;
8039
8040         bnxt_hwrm_fw_set_time(bp);
8041
8042         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
8043                            NETIF_F_TSO | NETIF_F_TSO6 |
8044                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
8045                            NETIF_F_GSO_IPXIP4 |
8046                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
8047                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
8048                            NETIF_F_RXCSUM | NETIF_F_GRO;
8049
8050         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
8051                 dev->hw_features |= NETIF_F_LRO;
8052
8053         dev->hw_enc_features =
8054                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
8055                         NETIF_F_TSO | NETIF_F_TSO6 |
8056                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
8057                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
8058                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
8059         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
8060                                     NETIF_F_GSO_GRE_CSUM;
8061         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
8062         dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
8063                             NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
8064         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
8065         dev->priv_flags |= IFF_UNICAST_FLT;
8066
8067         /* MTU range: 60 - 9500 */
8068         dev->min_mtu = ETH_ZLEN;
8069         dev->max_mtu = BNXT_MAX_MTU;
8070
8071 #ifdef CONFIG_BNXT_SRIOV
8072         init_waitqueue_head(&bp->sriov_cfg_wait);
8073         mutex_init(&bp->sriov_lock);
8074 #endif
8075         bp->gro_func = bnxt_gro_func_5730x;
8076         if (BNXT_CHIP_P4_PLUS(bp))
8077                 bp->gro_func = bnxt_gro_func_5731x;
8078         else
8079                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
8080
8081         rc = bnxt_hwrm_func_drv_rgtr(bp);
8082         if (rc)
8083                 goto init_err_pci_clean;
8084
8085         rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0);
8086         if (rc)
8087                 goto init_err_pci_clean;
8088
8089         bp->ulp_probe = bnxt_ulp_probe;
8090
8091         /* Get the MAX capabilities for this function */
8092         rc = bnxt_hwrm_func_qcaps(bp);
8093         if (rc) {
8094                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
8095                            rc);
8096                 rc = -1;
8097                 goto init_err_pci_clean;
8098         }
8099         rc = bnxt_init_mac_addr(bp);
8100         if (rc) {
8101                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
8102                 rc = -EADDRNOTAVAIL;
8103                 goto init_err_pci_clean;
8104         }
8105         rc = bnxt_hwrm_queue_qportcfg(bp);
8106         if (rc) {
8107                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n",
8108                            rc);
8109                 rc = -1;
8110                 goto init_err_pci_clean;
8111         }
8112
8113         bnxt_hwrm_func_qcfg(bp);
8114         bnxt_hwrm_port_led_qcaps(bp);
8115         bnxt_ethtool_init(bp);
8116         bnxt_dcb_init(bp);
8117
8118         rc = bnxt_probe_phy(bp);
8119         if (rc)
8120                 goto init_err_pci_clean;
8121
8122         bnxt_set_rx_skb_mode(bp, false);
8123         bnxt_set_tpa_flags(bp);
8124         bnxt_set_ring_params(bp);
8125         bnxt_set_max_func_irqs(bp, max_irqs);
8126         rc = bnxt_set_dflt_rings(bp, true);
8127         if (rc) {
8128                 netdev_err(bp->dev, "Not enough rings available.\n");
8129                 rc = -ENOMEM;
8130                 goto init_err_pci_clean;
8131         }
8132
8133         /* Default RSS hash cfg. */
8134         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
8135                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
8136                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
8137                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
8138         if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
8139                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
8140                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
8141                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
8142         }
8143
8144         bnxt_hwrm_vnic_qcaps(bp);
8145         if (bnxt_rfs_supported(bp)) {
8146                 dev->hw_features |= NETIF_F_NTUPLE;
8147                 if (bnxt_rfs_capable(bp)) {
8148                         bp->flags |= BNXT_FLAG_RFS;
8149                         dev->features |= NETIF_F_NTUPLE;
8150                 }
8151         }
8152
8153         if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
8154                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
8155
8156         rc = bnxt_init_int_mode(bp);
8157         if (rc)
8158                 goto init_err_pci_clean;
8159
8160         bnxt_get_wol_settings(bp);
8161         if (bp->flags & BNXT_FLAG_WOL_CAP)
8162                 device_set_wakeup_enable(&pdev->dev, bp->wol);
8163         else
8164                 device_set_wakeup_capable(&pdev->dev, false);
8165
8166         if (BNXT_PF(bp)) {
8167                 if (!bnxt_pf_wq) {
8168                         bnxt_pf_wq =
8169                                 create_singlethread_workqueue("bnxt_pf_wq");
8170                         if (!bnxt_pf_wq) {
8171                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
8172                                 goto init_err_pci_clean;
8173                         }
8174                 }
8175                 bnxt_init_tc(bp);
8176         }
8177
8178         rc = register_netdev(dev);
8179         if (rc)
8180                 goto init_err_cleanup_tc;
8181
8182         if (BNXT_PF(bp))
8183                 bnxt_dl_register(bp);
8184
8185         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
8186                     board_info[ent->driver_data].name,
8187                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
8188
8189         bnxt_parse_log_pcie_link(bp);
8190
8191         return 0;
8192
8193 init_err_cleanup_tc:
8194         bnxt_shutdown_tc(bp);
8195         bnxt_clear_int_mode(bp);
8196
8197 init_err_pci_clean:
8198         bnxt_cleanup_pci(bp);
8199
8200 init_err_free:
8201         free_netdev(dev);
8202         return rc;
8203 }
8204
8205 static void bnxt_shutdown(struct pci_dev *pdev)
8206 {
8207         struct net_device *dev = pci_get_drvdata(pdev);
8208         struct bnxt *bp;
8209
8210         if (!dev)
8211                 return;
8212
8213         rtnl_lock();
8214         bp = netdev_priv(dev);
8215         if (!bp)
8216                 goto shutdown_exit;
8217
8218         if (netif_running(dev))
8219                 dev_close(dev);
8220
8221         if (system_state == SYSTEM_POWER_OFF) {
8222                 bnxt_ulp_shutdown(bp);
8223                 bnxt_clear_int_mode(bp);
8224                 pci_wake_from_d3(pdev, bp->wol);
8225                 pci_set_power_state(pdev, PCI_D3hot);
8226         }
8227
8228 shutdown_exit:
8229         rtnl_unlock();
8230 }
8231
8232 #ifdef CONFIG_PM_SLEEP
8233 static int bnxt_suspend(struct device *device)
8234 {
8235         struct pci_dev *pdev = to_pci_dev(device);
8236         struct net_device *dev = pci_get_drvdata(pdev);
8237         struct bnxt *bp = netdev_priv(dev);
8238         int rc = 0;
8239
8240         rtnl_lock();
8241         if (netif_running(dev)) {
8242                 netif_device_detach(dev);
8243                 rc = bnxt_close(dev);
8244         }
8245         bnxt_hwrm_func_drv_unrgtr(bp);
8246         rtnl_unlock();
8247         return rc;
8248 }
8249
8250 static int bnxt_resume(struct device *device)
8251 {
8252         struct pci_dev *pdev = to_pci_dev(device);
8253         struct net_device *dev = pci_get_drvdata(pdev);
8254         struct bnxt *bp = netdev_priv(dev);
8255         int rc = 0;
8256
8257         rtnl_lock();
8258         if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) {
8259                 rc = -ENODEV;
8260                 goto resume_exit;
8261         }
8262         rc = bnxt_hwrm_func_reset(bp);
8263         if (rc) {
8264                 rc = -EBUSY;
8265                 goto resume_exit;
8266         }
8267         bnxt_get_wol_settings(bp);
8268         if (netif_running(dev)) {
8269                 rc = bnxt_open(dev);
8270                 if (!rc)
8271                         netif_device_attach(dev);
8272         }
8273
8274 resume_exit:
8275         rtnl_unlock();
8276         return rc;
8277 }
8278
8279 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
8280 #define BNXT_PM_OPS (&bnxt_pm_ops)
8281
8282 #else
8283
8284 #define BNXT_PM_OPS NULL
8285
8286 #endif /* CONFIG_PM_SLEEP */
8287
8288 /**
8289  * bnxt_io_error_detected - called when PCI error is detected
8290  * @pdev: Pointer to PCI device
8291  * @state: The current pci connection state
8292  *
8293  * This function is called after a PCI bus error affecting
8294  * this device has been detected.
8295  */
8296 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
8297                                                pci_channel_state_t state)
8298 {
8299         struct net_device *netdev = pci_get_drvdata(pdev);
8300         struct bnxt *bp = netdev_priv(netdev);
8301
8302         netdev_info(netdev, "PCI I/O error detected\n");
8303
8304         rtnl_lock();
8305         netif_device_detach(netdev);
8306
8307         bnxt_ulp_stop(bp);
8308
8309         if (state == pci_channel_io_perm_failure) {
8310                 rtnl_unlock();
8311                 return PCI_ERS_RESULT_DISCONNECT;
8312         }
8313
8314         if (netif_running(netdev))
8315                 bnxt_close(netdev);
8316
8317         pci_disable_device(pdev);
8318         rtnl_unlock();
8319
8320         /* Request a slot slot reset. */
8321         return PCI_ERS_RESULT_NEED_RESET;
8322 }
8323
8324 /**
8325  * bnxt_io_slot_reset - called after the pci bus has been reset.
8326  * @pdev: Pointer to PCI device
8327  *
8328  * Restart the card from scratch, as if from a cold-boot.
8329  * At this point, the card has exprienced a hard reset,
8330  * followed by fixups by BIOS, and has its config space
8331  * set up identically to what it was at cold boot.
8332  */
8333 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
8334 {
8335         struct net_device *netdev = pci_get_drvdata(pdev);
8336         struct bnxt *bp = netdev_priv(netdev);
8337         int err = 0;
8338         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
8339
8340         netdev_info(bp->dev, "PCI Slot Reset\n");
8341
8342         rtnl_lock();
8343
8344         if (pci_enable_device(pdev)) {
8345                 dev_err(&pdev->dev,
8346                         "Cannot re-enable PCI device after reset.\n");
8347         } else {
8348                 pci_set_master(pdev);
8349
8350                 err = bnxt_hwrm_func_reset(bp);
8351                 if (!err && netif_running(netdev))
8352                         err = bnxt_open(netdev);
8353
8354                 if (!err) {
8355                         result = PCI_ERS_RESULT_RECOVERED;
8356                         bnxt_ulp_start(bp);
8357                 }
8358         }
8359
8360         if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
8361                 dev_close(netdev);
8362
8363         rtnl_unlock();
8364
8365         err = pci_cleanup_aer_uncorrect_error_status(pdev);
8366         if (err) {
8367                 dev_err(&pdev->dev,
8368                         "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
8369                          err); /* non-fatal, continue */
8370         }
8371
8372         return PCI_ERS_RESULT_RECOVERED;
8373 }
8374
8375 /**
8376  * bnxt_io_resume - called when traffic can start flowing again.
8377  * @pdev: Pointer to PCI device
8378  *
8379  * This callback is called when the error recovery driver tells
8380  * us that its OK to resume normal operation.
8381  */
8382 static void bnxt_io_resume(struct pci_dev *pdev)
8383 {
8384         struct net_device *netdev = pci_get_drvdata(pdev);
8385
8386         rtnl_lock();
8387
8388         netif_device_attach(netdev);
8389
8390         rtnl_unlock();
8391 }
8392
8393 static const struct pci_error_handlers bnxt_err_handler = {
8394         .error_detected = bnxt_io_error_detected,
8395         .slot_reset     = bnxt_io_slot_reset,
8396         .resume         = bnxt_io_resume
8397 };
8398
8399 static struct pci_driver bnxt_pci_driver = {
8400         .name           = DRV_MODULE_NAME,
8401         .id_table       = bnxt_pci_tbl,
8402         .probe          = bnxt_init_one,
8403         .remove         = bnxt_remove_one,
8404         .shutdown       = bnxt_shutdown,
8405         .driver.pm      = BNXT_PM_OPS,
8406         .err_handler    = &bnxt_err_handler,
8407 #if defined(CONFIG_BNXT_SRIOV)
8408         .sriov_configure = bnxt_sriov_configure,
8409 #endif
8410 };
8411
8412 static int __init bnxt_init(void)
8413 {
8414         return pci_register_driver(&bnxt_pci_driver);
8415 }
8416
8417 static void __exit bnxt_exit(void)
8418 {
8419         pci_unregister_driver(&bnxt_pci_driver);
8420         if (bnxt_pf_wq)
8421                 destroy_workqueue(bnxt_pf_wq);
8422 }
8423
8424 module_init(bnxt_init);
8425 module_exit(bnxt_exit);