drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2019 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/mdio.h>
  35 #include <linux/if.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/if_bridge.h>
  38 #include <linux/rtc.h>
  39 #include <linux/bpf.h>
  40 #include <net/ip.h>
  41 #include <net/tcp.h>
  42 #include <net/udp.h>
  43 #include <net/checksum.h>
  44 #include <net/ip6_checksum.h>
  45 #include <net/udp_tunnel.h>
  46 #include <linux/workqueue.h>
  47 #include <linux/prefetch.h>
  48 #include <linux/cache.h>
  49 #include <linux/log2.h>
  50 #include <linux/aer.h>
  51 #include <linux/bitmap.h>
  52 #include <linux/cpu_rmap.h>
  53 #include <linux/cpumask.h>
  54 #include <net/pkt_cls.h>
  55 #include <linux/hwmon.h>
  56 #include <linux/hwmon-sysfs.h>
  57 #include <net/page_pool.h>
  58
  59 #include "bnxt_hsi.h"
  60 #include "bnxt.h"
  61 #include "bnxt_ulp.h"
  62 #include "bnxt_sriov.h"
  63 #include "bnxt_ethtool.h"
  64 #include "bnxt_dcb.h"
  65 #include "bnxt_xdp.h"
  66 #include "bnxt_vfr.h"
  67 #include "bnxt_tc.h"
  68 #include "bnxt_devlink.h"
  69 #include "bnxt_debugfs.h"
  70
  71 #define BNXT_TX_TIMEOUT         (5 * HZ)
  72 #define BNXT_DEF_MSG_ENABLE     (NETIF_MSG_DRV | NETIF_MSG_HW)
  73
  74 MODULE_LICENSE("GPL");
  75 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  76
  77 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  78 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  79 #define BNXT_RX_COPY_THRESH 256
  80
  81 #define BNXT_TX_PUSH_THRESH 164
  82
  83 enum board_idx {
  84         BCM57301,
  85         BCM57302,
  86         BCM57304,
  87         BCM57417_NPAR,
  88         BCM58700,
  89         BCM57311,
  90         BCM57312,
  91         BCM57402,
  92         BCM57404,
  93         BCM57406,
  94         BCM57402_NPAR,
  95         BCM57407,
  96         BCM57412,
  97         BCM57414,
  98         BCM57416,
  99         BCM57417,
 100         BCM57412_NPAR,
 101         BCM57314,
 102         BCM57417_SFP,
 103         BCM57416_SFP,
 104         BCM57404_NPAR,
 105         BCM57406_NPAR,
 106         BCM57407_SFP,
 107         BCM57407_NPAR,
 108         BCM57414_NPAR,
 109         BCM57416_NPAR,
 110         BCM57452,
 111         BCM57454,
 112         BCM5745x_NPAR,
 113         BCM57508,
 114         BCM57504,
 115         BCM57502,
 116         BCM57508_NPAR,
 117         BCM57504_NPAR,
 118         BCM57502_NPAR,
 119         BCM58802,
 120         BCM58804,
 121         BCM58808,
 122         NETXTREME_E_VF,
 123         NETXTREME_C_VF,
 124         NETXTREME_S_VF,
 125         NETXTREME_E_P5_VF,
 126 };
 127
 128 /* indexed by enum above */
 129 static const struct {
 130         char *name;
 131 } board_info[] = {
 132         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
 133         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
 134         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 135         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
 136         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
 137         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
 138         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
 139         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 140         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 141         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 142         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 143         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 144         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 145         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 146         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 147         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 148         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 149         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 150         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 151         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 152         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 153         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 154         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 155         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 156         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 157         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 158         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 159         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 160         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 161         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 162         [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 163         [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
 164         [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
 165         [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
 166         [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
 167         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 168         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 169         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 170         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 171         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 172         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 173         [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
 174 };
 175
 176 static const struct pci_device_id bnxt_pci_tbl[] = {
 177         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 178         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 179         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 180         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 181         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 182         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 183         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 184         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 185         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 186         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 187         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 188         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 189         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 190         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 191         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 192         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 193         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 194         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 195         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 196         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 197         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 198         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 199         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 200         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 201         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 202         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 203         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 204         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 205         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 206         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 207         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 208         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 209         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 210         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 211         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 212         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 213         { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
 214         { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
 215         { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
 216         { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
 217         { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
 218         { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
 219         { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
 220         { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
 221         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 222         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 223 #ifdef CONFIG_BNXT_SRIOV
 224         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 225         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 226         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 227         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 228         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 229         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 230         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 231         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 232         { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
 233         { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
 234         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 235 #endif
 236         { 0 }
 237 };
 238
 239 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 240
 241 static const u16 bnxt_vf_req_snif[] = {
 242         HWRM_FUNC_CFG,
 243         HWRM_FUNC_VF_CFG,
 244         HWRM_PORT_PHY_QCFG,
 245         HWRM_CFA_L2_FILTER_ALLOC,
 246 };
 247
 248 static const u16 bnxt_async_events_arr[] = {
 249         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 250         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
 251         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 252         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 253         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 254         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 255         ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
 256         ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
 257         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
 258         ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
 259 };
 260
 261 static struct workqueue_struct *bnxt_pf_wq;
 262
 263 static bool bnxt_vf_pciid(enum board_idx idx)
 264 {
 265         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 266                 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
 267 }
 268
 269 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 270 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 271 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 272
 273 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 274                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 275
 276 #define BNXT_DB_CQ(db, idx)                                             \
 277         writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
 278
 279 #define BNXT_DB_NQ_P5(db, idx)                                          \
 280         writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
 281
 282 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 283         writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
 284
 285 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 286         writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
 287
 288 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 289 {
 290         if (bp->flags & BNXT_FLAG_CHIP_P5)
 291                 BNXT_DB_NQ_P5(db, idx);
 292         else
 293                 BNXT_DB_CQ(db, idx);
 294 }
 295
 296 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 297 {
 298         if (bp->flags & BNXT_FLAG_CHIP_P5)
 299                 BNXT_DB_NQ_ARM_P5(db, idx);
 300         else
 301                 BNXT_DB_CQ_ARM(db, idx);
 302 }
 303
 304 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 305 {
 306         if (bp->flags & BNXT_FLAG_CHIP_P5)
 307                 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
 308                        db->doorbell);
 309         else
 310                 BNXT_DB_CQ(db, idx);
 311 }
 312
 313 const u16 bnxt_lhint_arr[] = {
 314         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 315         TX_BD_FLAGS_LHINT_512_TO_1023,
 316         TX_BD_FLAGS_LHINT_1024_TO_2047,
 317         TX_BD_FLAGS_LHINT_1024_TO_2047,
 318         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 319         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 320         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 321         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 322         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 323         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 324         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 325         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 326         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 327         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 328         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 329         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 330         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 331         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 332         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 333 };
 334
 335 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 336 {
 337         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 338
 339         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 340                 return 0;
 341
 342         return md_dst->u.port_info.port_id;
 343 }
 344
 345 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 346 {
 347         struct bnxt *bp = netdev_priv(dev);
 348         struct tx_bd *txbd;
 349         struct tx_bd_ext *txbd1;
 350         struct netdev_queue *txq;
 351         int i;
 352         dma_addr_t mapping;
 353         unsigned int length, pad = 0;
 354         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 355         u16 prod, last_frag;
 356         struct pci_dev *pdev = bp->pdev;
 357         struct bnxt_tx_ring_info *txr;
 358         struct bnxt_sw_tx_bd *tx_buf;
 359
 360         i = skb_get_queue_mapping(skb);
 361         if (unlikely(i >= bp->tx_nr_rings)) {
 362                 dev_kfree_skb_any(skb);
 363                 return NETDEV_TX_OK;
 364         }
 365
 366         txq = netdev_get_tx_queue(dev, i);
 367         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 368         prod = txr->tx_prod;
 369
 370         free_size = bnxt_tx_avail(bp, txr);
 371         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 372                 netif_tx_stop_queue(txq);
 373                 return NETDEV_TX_BUSY;
 374         }
 375
 376         length = skb->len;
 377         len = skb_headlen(skb);
 378         last_frag = skb_shinfo(skb)->nr_frags;
 379
 380         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 381
 382         txbd->tx_bd_opaque = prod;
 383
 384         tx_buf = &txr->tx_buf_ring[prod];
 385         tx_buf->skb = skb;
 386         tx_buf->nr_frags = last_frag;
 387
 388         vlan_tag_flags = 0;
 389         cfa_action = bnxt_xmit_get_cfa_action(skb);
 390         if (skb_vlan_tag_present(skb)) {
 391                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 392                                  skb_vlan_tag_get(skb);
 393                 /* Currently supports 8021Q, 8021AD vlan offloads
 394                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 395                  */
 396                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 397                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 398         }
 399
 400         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 401                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 402                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 403                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 404                 void __iomem *db = txr->tx_db.doorbell;
 405                 void *pdata = tx_push_buf->data;
 406                 u64 *end;
 407                 int j, push_len;
 408
 409                 /* Set COAL_NOW to be ready quickly for the next push */
 410                 tx_push->tx_bd_len_flags_type =
 411                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 412                                         TX_BD_TYPE_LONG_TX_BD |
 413                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 414                                         TX_BD_FLAGS_COAL_NOW |
 415                                         TX_BD_FLAGS_PACKET_END |
 416                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 417
 418                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 419                         tx_push1->tx_bd_hsize_lflags =
 420                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 421                 else
 422                         tx_push1->tx_bd_hsize_lflags = 0;
 423
 424                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 425                 tx_push1->tx_bd_cfa_action =
 426                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 427
 428                 end = pdata + length;
 429                 end = PTR_ALIGN(end, 8) - 1;
 430                 *end = 0;
 431
 432                 skb_copy_from_linear_data(skb, pdata, len);
 433                 pdata += len;
 434                 for (j = 0; j < last_frag; j++) {
 435                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 436                         void *fptr;
 437
 438                         fptr = skb_frag_address_safe(frag);
 439                         if (!fptr)
 440                                 goto normal_tx;
 441
 442                         memcpy(pdata, fptr, skb_frag_size(frag));
 443                         pdata += skb_frag_size(frag);
 444                 }
 445
 446                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 447                 txbd->tx_bd_haddr = txr->data_mapping;
 448                 prod = NEXT_TX(prod);
 449                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 450                 memcpy(txbd, tx_push1, sizeof(*txbd));
 451                 prod = NEXT_TX(prod);
 452                 tx_push->doorbell =
 453                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 454                 txr->tx_prod = prod;
 455
 456                 tx_buf->is_push = 1;
 457                 netdev_tx_sent_queue(txq, skb->len);
 458                 wmb();  /* Sync is_push and byte queue before pushing data */
 459
 460                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 461                 if (push_len > 16) {
 462                         __iowrite64_copy(db, tx_push_buf, 16);
 463                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 464                                          (push_len - 16) << 1);
 465                 } else {
 466                         __iowrite64_copy(db, tx_push_buf, push_len);
 467                 }
 468
 469                 goto tx_done;
 470         }
 471
 472 normal_tx:
 473         if (length < BNXT_MIN_PKT_SIZE) {
 474                 pad = BNXT_MIN_PKT_SIZE - length;
 475                 if (skb_pad(skb, pad)) {
 476                         /* SKB already freed. */
 477                         tx_buf->skb = NULL;
 478                         return NETDEV_TX_OK;
 479                 }
 480                 length = BNXT_MIN_PKT_SIZE;
 481         }
 482
 483         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 484
 485         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 486                 dev_kfree_skb_any(skb);
 487                 tx_buf->skb = NULL;
 488                 return NETDEV_TX_OK;
 489         }
 490
 491         dma_unmap_addr_set(tx_buf, mapping, mapping);
 492         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 493                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 494
 495         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 496
 497         prod = NEXT_TX(prod);
 498         txbd1 = (struct tx_bd_ext *)
 499                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 500
 501         txbd1->tx_bd_hsize_lflags = 0;
 502         if (skb_is_gso(skb)) {
 503                 u32 hdr_len;
 504
 505                 if (skb->encapsulation)
 506                         hdr_len = skb_inner_network_offset(skb) +
 507                                 skb_inner_network_header_len(skb) +
 508                                 inner_tcp_hdrlen(skb);
 509                 else
 510                         hdr_len = skb_transport_offset(skb) +
 511                                 tcp_hdrlen(skb);
 512
 513                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 514                                         TX_BD_FLAGS_T_IPID |
 515                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 516                 length = skb_shinfo(skb)->gso_size;
 517                 txbd1->tx_bd_mss = cpu_to_le32(length);
 518                 length += hdr_len;
 519         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 520                 txbd1->tx_bd_hsize_lflags =
 521                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 522                 txbd1->tx_bd_mss = 0;
 523         }
 524
 525         length >>= 9;
 526         if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
 527                 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
 528                                      skb->len);
 529                 i = 0;
 530                 goto tx_dma_error;
 531         }
 532         flags |= bnxt_lhint_arr[length];
 533         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 534
 535         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 536         txbd1->tx_bd_cfa_action =
 537                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 538         for (i = 0; i < last_frag; i++) {
 539                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 540
 541                 prod = NEXT_TX(prod);
 542                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 543
 544                 len = skb_frag_size(frag);
 545                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 546                                            DMA_TO_DEVICE);
 547
 548                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 549                         goto tx_dma_error;
 550
 551                 tx_buf = &txr->tx_buf_ring[prod];
 552                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 553
 554                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 555
 556                 flags = len << TX_BD_LEN_SHIFT;
 557                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 558         }
 559
 560         flags &= ~TX_BD_LEN;
 561         txbd->tx_bd_len_flags_type =
 562                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 563                             TX_BD_FLAGS_PACKET_END);
 564
 565         netdev_tx_sent_queue(txq, skb->len);
 566
 567         /* Sync BD data before updating doorbell */
 568         wmb();
 569
 570         prod = NEXT_TX(prod);
 571         txr->tx_prod = prod;
 572
 573         if (!netdev_xmit_more() || netif_xmit_stopped(txq))
 574                 bnxt_db_write(bp, &txr->tx_db, prod);
 575
 576 tx_done:
 577
 578         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 579                 if (netdev_xmit_more() && !tx_buf->is_push)
 580                         bnxt_db_write(bp, &txr->tx_db, prod);
 581
 582                 netif_tx_stop_queue(txq);
 583
 584                 /* netif_tx_stop_queue() must be done before checking
 585                  * tx index in bnxt_tx_avail() below, because in
 586                  * bnxt_tx_int(), we update tx index before checking for
 587                  * netif_tx_queue_stopped().
 588                  */
 589                 smp_mb();
 590                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 591                         netif_tx_wake_queue(txq);
 592         }
 593         return NETDEV_TX_OK;
 594
 595 tx_dma_error:
 596         last_frag = i;
 597
 598         /* start back at beginning and unmap skb */
 599         prod = txr->tx_prod;
 600         tx_buf = &txr->tx_buf_ring[prod];
 601         tx_buf->skb = NULL;
 602         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 603                          skb_headlen(skb), PCI_DMA_TODEVICE);
 604         prod = NEXT_TX(prod);
 605
 606         /* unmap remaining mapped pages */
 607         for (i = 0; i < last_frag; i++) {
 608                 prod = NEXT_TX(prod);
 609                 tx_buf = &txr->tx_buf_ring[prod];
 610                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 611                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 612                                PCI_DMA_TODEVICE);
 613         }
 614
 615         dev_kfree_skb_any(skb);
 616         return NETDEV_TX_OK;
 617 }
 618
 619 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 620 {
 621         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 622         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 623         u16 cons = txr->tx_cons;
 624         struct pci_dev *pdev = bp->pdev;
 625         int i;
 626         unsigned int tx_bytes = 0;
 627
 628         for (i = 0; i < nr_pkts; i++) {
 629                 struct bnxt_sw_tx_bd *tx_buf;
 630                 struct sk_buff *skb;
 631                 int j, last;
 632
 633                 tx_buf = &txr->tx_buf_ring[cons];
 634                 cons = NEXT_TX(cons);
 635                 skb = tx_buf->skb;
 636                 tx_buf->skb = NULL;
 637
 638                 if (tx_buf->is_push) {
 639                         tx_buf->is_push = 0;
 640                         goto next_tx_int;
 641                 }
 642
 643                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 644                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 645                 last = tx_buf->nr_frags;
 646
 647                 for (j = 0; j < last; j++) {
 648                         cons = NEXT_TX(cons);
 649                         tx_buf = &txr->tx_buf_ring[cons];
 650                         dma_unmap_page(
 651                                 &pdev->dev,
 652                                 dma_unmap_addr(tx_buf, mapping),
 653                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 654                                 PCI_DMA_TODEVICE);
 655                 }
 656
 657 next_tx_int:
 658                 cons = NEXT_TX(cons);
 659
 660                 tx_bytes += skb->len;
 661                 dev_kfree_skb_any(skb);
 662         }
 663
 664         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 665         txr->tx_cons = cons;
 666
 667         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 668          * before checking for netif_tx_queue_stopped().  Without the
 669          * memory barrier, there is a small possibility that bnxt_start_xmit()
 670          * will miss it and cause the queue to be stopped forever.
 671          */
 672         smp_mb();
 673
 674         if (unlikely(netif_tx_queue_stopped(txq)) &&
 675             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 676                 __netif_tx_lock(txq, smp_processor_id());
 677                 if (netif_tx_queue_stopped(txq) &&
 678                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 679                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 680                         netif_tx_wake_queue(txq);
 681                 __netif_tx_unlock(txq);
 682         }
 683 }
 684
 685 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 686                                          struct bnxt_rx_ring_info *rxr,
 687                                          gfp_t gfp)
 688 {
 689         struct device *dev = &bp->pdev->dev;
 690         struct page *page;
 691
 692         page = page_pool_dev_alloc_pages(rxr->page_pool);
 693         if (!page)
 694                 return NULL;
 695
 696         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 697                                       DMA_ATTR_WEAK_ORDERING);
 698         if (dma_mapping_error(dev, *mapping)) {
 699                 page_pool_recycle_direct(rxr->page_pool, page);
 700                 return NULL;
 701         }
 702         *mapping += bp->rx_dma_offset;
 703         return page;
 704 }
 705
 706 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 707                                        gfp_t gfp)
 708 {
 709         u8 *data;
 710         struct pci_dev *pdev = bp->pdev;
 711
 712         data = kmalloc(bp->rx_buf_size, gfp);
 713         if (!data)
 714                 return NULL;
 715
 716         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 717                                         bp->rx_buf_use_size, bp->rx_dir,
 718                                         DMA_ATTR_WEAK_ORDERING);
 719
 720         if (dma_mapping_error(&pdev->dev, *mapping)) {
 721                 kfree(data);
 722                 data = NULL;
 723         }
 724         return data;
 725 }
 726
 727 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 728                        u16 prod, gfp_t gfp)
 729 {
 730         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 731         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 732         dma_addr_t mapping;
 733
 734         if (BNXT_RX_PAGE_MODE(bp)) {
 735                 struct page *page =
 736                         __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 737
 738                 if (!page)
 739                         return -ENOMEM;
 740
 741                 rx_buf->data = page;
 742                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 743         } else {
 744                 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 745
 746                 if (!data)
 747                         return -ENOMEM;
 748
 749                 rx_buf->data = data;
 750                 rx_buf->data_ptr = data + bp->rx_offset;
 751         }
 752         rx_buf->mapping = mapping;
 753
 754         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 755         return 0;
 756 }
 757
 758 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 759 {
 760         u16 prod = rxr->rx_prod;
 761         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 762         struct rx_bd *cons_bd, *prod_bd;
 763
 764         prod_rx_buf = &rxr->rx_buf_ring[prod];
 765         cons_rx_buf = &rxr->rx_buf_ring[cons];
 766
 767         prod_rx_buf->data = data;
 768         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 769
 770         prod_rx_buf->mapping = cons_rx_buf->mapping;
 771
 772         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 773         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 774
 775         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 776 }
 777
 778 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 779 {
 780         u16 next, max = rxr->rx_agg_bmap_size;
 781
 782         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 783         if (next >= max)
 784                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 785         return next;
 786 }
 787
 788 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 789                                      struct bnxt_rx_ring_info *rxr,
 790                                      u16 prod, gfp_t gfp)
 791 {
 792         struct rx_bd *rxbd =
 793                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 794         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 795         struct pci_dev *pdev = bp->pdev;
 796         struct page *page;
 797         dma_addr_t mapping;
 798         u16 sw_prod = rxr->rx_sw_agg_prod;
 799         unsigned int offset = 0;
 800
 801         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 802                 page = rxr->rx_page;
 803                 if (!page) {
 804                         page = alloc_page(gfp);
 805                         if (!page)
 806                                 return -ENOMEM;
 807                         rxr->rx_page = page;
 808                         rxr->rx_page_offset = 0;
 809                 }
 810                 offset = rxr->rx_page_offset;
 811                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 812                 if (rxr->rx_page_offset == PAGE_SIZE)
 813                         rxr->rx_page = NULL;
 814                 else
 815                         get_page(page);
 816         } else {
 817                 page = alloc_page(gfp);
 818                 if (!page)
 819                         return -ENOMEM;
 820         }
 821
 822         mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 823                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
 824                                      DMA_ATTR_WEAK_ORDERING);
 825         if (dma_mapping_error(&pdev->dev, mapping)) {
 826                 __free_page(page);
 827                 return -EIO;
 828         }
 829
 830         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 831                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 832
 833         __set_bit(sw_prod, rxr->rx_agg_bmap);
 834         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 835         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 836
 837         rx_agg_buf->page = page;
 838         rx_agg_buf->offset = offset;
 839         rx_agg_buf->mapping = mapping;
 840         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 841         rxbd->rx_bd_opaque = sw_prod;
 842         return 0;
 843 }
 844
 845 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
 846                                        struct bnxt_cp_ring_info *cpr,
 847                                        u16 cp_cons, u16 curr)
 848 {
 849         struct rx_agg_cmp *agg;
 850
 851         cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
 852         agg = (struct rx_agg_cmp *)
 853                 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 854         return agg;
 855 }
 856
 857 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
 858                                               struct bnxt_rx_ring_info *rxr,
 859                                               u16 agg_id, u16 curr)
 860 {
 861         struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
 862
 863         return &tpa_info->agg_arr[curr];
 864 }
 865
 866 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
 867                                    u16 start, u32 agg_bufs, bool tpa)
 868 {
 869         struct bnxt_napi *bnapi = cpr->bnapi;
 870         struct bnxt *bp = bnapi->bp;
 871         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 872         u16 prod = rxr->rx_agg_prod;
 873         u16 sw_prod = rxr->rx_sw_agg_prod;
 874         bool p5_tpa = false;
 875         u32 i;
 876
 877         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
 878                 p5_tpa = true;
 879
 880         for (i = 0; i < agg_bufs; i++) {
 881                 u16 cons;
 882                 struct rx_agg_cmp *agg;
 883                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 884                 struct rx_bd *prod_bd;
 885                 struct page *page;
 886
 887                 if (p5_tpa)
 888                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
 889                 else
 890                         agg = bnxt_get_agg(bp, cpr, idx, start + i);
 891                 cons = agg->rx_agg_cmp_opaque;
 892                 __clear_bit(cons, rxr->rx_agg_bmap);
 893
 894                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 895                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 896
 897                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 898                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 899                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 900
 901                 /* It is possible for sw_prod to be equal to cons, so
 902                  * set cons_rx_buf->page to NULL first.
 903                  */
 904                 page = cons_rx_buf->page;
 905                 cons_rx_buf->page = NULL;
 906                 prod_rx_buf->page = page;
 907                 prod_rx_buf->offset = cons_rx_buf->offset;
 908
 909                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 910
 911                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 912
 913                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 914                 prod_bd->rx_bd_opaque = sw_prod;
 915
 916                 prod = NEXT_RX_AGG(prod);
 917                 sw_prod = NEXT_RX_AGG(sw_prod);
 918         }
 919         rxr->rx_agg_prod = prod;
 920         rxr->rx_sw_agg_prod = sw_prod;
 921 }
 922
 923 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 924                                         struct bnxt_rx_ring_info *rxr,
 925                                         u16 cons, void *data, u8 *data_ptr,
 926                                         dma_addr_t dma_addr,
 927                                         unsigned int offset_and_len)
 928 {
 929         unsigned int payload = offset_and_len >> 16;
 930         unsigned int len = offset_and_len & 0xffff;
 931         skb_frag_t *frag;
 932         struct page *page = data;
 933         u16 prod = rxr->rx_prod;
 934         struct sk_buff *skb;
 935         int off, err;
 936
 937         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 938         if (unlikely(err)) {
 939                 bnxt_reuse_rx_data(rxr, cons, data);
 940                 return NULL;
 941         }
 942         dma_addr -= bp->rx_dma_offset;
 943         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 944                              DMA_ATTR_WEAK_ORDERING);
 945         page_pool_release_page(rxr->page_pool, page);
 946
 947         if (unlikely(!payload))
 948                 payload = eth_get_headlen(bp->dev, data_ptr, len);
 949
 950         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
 951         if (!skb) {
 952                 __free_page(page);
 953                 return NULL;
 954         }
 955
 956         off = (void *)data_ptr - page_address(page);
 957         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
 958         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
 959                payload + NET_IP_ALIGN);
 960
 961         frag = &skb_shinfo(skb)->frags[0];
 962         skb_frag_size_sub(frag, payload);
 963         skb_frag_off_add(frag, payload);
 964         skb->data_len -= payload;
 965         skb->tail += payload;
 966
 967         return skb;
 968 }
 969
 970 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 971                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 972                                    void *data, u8 *data_ptr,
 973                                    dma_addr_t dma_addr,
 974                                    unsigned int offset_and_len)
 975 {
 976         u16 prod = rxr->rx_prod;
 977         struct sk_buff *skb;
 978         int err;
 979
 980         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 981         if (unlikely(err)) {
 982                 bnxt_reuse_rx_data(rxr, cons, data);
 983                 return NULL;
 984         }
 985
 986         skb = build_skb(data, 0);
 987         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 988                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
 989         if (!skb) {
 990                 kfree(data);
 991                 return NULL;
 992         }
 993
 994         skb_reserve(skb, bp->rx_offset);
 995         skb_put(skb, offset_and_len & 0xffff);
 996         return skb;
 997 }
 998
 999 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
1000                                      struct bnxt_cp_ring_info *cpr,
1001                                      struct sk_buff *skb, u16 idx,
1002                                      u32 agg_bufs, bool tpa)
1003 {
1004         struct bnxt_napi *bnapi = cpr->bnapi;
1005         struct pci_dev *pdev = bp->pdev;
1006         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1007         u16 prod = rxr->rx_agg_prod;
1008         bool p5_tpa = false;
1009         u32 i;
1010
1011         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1012                 p5_tpa = true;
1013
1014         for (i = 0; i < agg_bufs; i++) {
1015                 u16 cons, frag_len;
1016                 struct rx_agg_cmp *agg;
1017                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1018                 struct page *page;
1019                 dma_addr_t mapping;
1020
1021                 if (p5_tpa)
1022                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1023                 else
1024                         agg = bnxt_get_agg(bp, cpr, idx, i);
1025                 cons = agg->rx_agg_cmp_opaque;
1026                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1027                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1028
1029                 cons_rx_buf = &rxr->rx_agg_ring[cons];
1030                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
1031                                    cons_rx_buf->offset, frag_len);
1032                 __clear_bit(cons, rxr->rx_agg_bmap);
1033
1034                 /* It is possible for bnxt_alloc_rx_page() to allocate
1035                  * a sw_prod index that equals the cons index, so we
1036                  * need to clear the cons entry now.
1037                  */
1038                 mapping = cons_rx_buf->mapping;
1039                 page = cons_rx_buf->page;
1040                 cons_rx_buf->page = NULL;
1041
1042                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1043                         struct skb_shared_info *shinfo;
1044                         unsigned int nr_frags;
1045
1046                         shinfo = skb_shinfo(skb);
1047                         nr_frags = --shinfo->nr_frags;
1048                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1049
1050                         dev_kfree_skb(skb);
1051
1052                         cons_rx_buf->page = page;
1053
1054                         /* Update prod since possibly some pages have been
1055                          * allocated already.
1056                          */
1057                         rxr->rx_agg_prod = prod;
1058                         bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1059                         return NULL;
1060                 }
1061
1062                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1063                                      PCI_DMA_FROMDEVICE,
1064                                      DMA_ATTR_WEAK_ORDERING);
1065
1066                 skb->data_len += frag_len;
1067                 skb->len += frag_len;
1068                 skb->truesize += PAGE_SIZE;
1069
1070                 prod = NEXT_RX_AGG(prod);
1071         }
1072         rxr->rx_agg_prod = prod;
1073         return skb;
1074 }
1075
1076 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1077                                u8 agg_bufs, u32 *raw_cons)
1078 {
1079         u16 last;
1080         struct rx_agg_cmp *agg;
1081
1082         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1083         last = RING_CMP(*raw_cons);
1084         agg = (struct rx_agg_cmp *)
1085                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1086         return RX_AGG_CMP_VALID(agg, *raw_cons);
1087 }
1088
1089 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1090                                             unsigned int len,
1091                                             dma_addr_t mapping)
1092 {
1093         struct bnxt *bp = bnapi->bp;
1094         struct pci_dev *pdev = bp->pdev;
1095         struct sk_buff *skb;
1096
1097         skb = napi_alloc_skb(&bnapi->napi, len);
1098         if (!skb)
1099                 return NULL;
1100
1101         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1102                                 bp->rx_dir);
1103
1104         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1105                len + NET_IP_ALIGN);
1106
1107         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1108                                    bp->rx_dir);
1109
1110         skb_put(skb, len);
1111         return skb;
1112 }
1113
1114 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1115                            u32 *raw_cons, void *cmp)
1116 {
1117         struct rx_cmp *rxcmp = cmp;
1118         u32 tmp_raw_cons = *raw_cons;
1119         u8 cmp_type, agg_bufs = 0;
1120
1121         cmp_type = RX_CMP_TYPE(rxcmp);
1122
1123         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1124                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1125                             RX_CMP_AGG_BUFS) >>
1126                            RX_CMP_AGG_BUFS_SHIFT;
1127         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1128                 struct rx_tpa_end_cmp *tpa_end = cmp;
1129
1130                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1131                         return 0;
1132
1133                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1134         }
1135
1136         if (agg_bufs) {
1137                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1138                         return -EBUSY;
1139         }
1140         *raw_cons = tmp_raw_cons;
1141         return 0;
1142 }
1143
1144 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1145 {
1146         if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
1147                 return;
1148
1149         if (BNXT_PF(bp))
1150                 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1151         else
1152                 schedule_delayed_work(&bp->fw_reset_task, delay);
1153 }
1154
1155 static void bnxt_queue_sp_work(struct bnxt *bp)
1156 {
1157         if (BNXT_PF(bp))
1158                 queue_work(bnxt_pf_wq, &bp->sp_task);
1159         else
1160                 schedule_work(&bp->sp_task);
1161 }
1162
1163 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1164 {
1165         if (!rxr->bnapi->in_reset) {
1166                 rxr->bnapi->in_reset = true;
1167                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1168                         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1169                 else
1170                         set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
1171                 bnxt_queue_sp_work(bp);
1172         }
1173         rxr->rx_next_cons = 0xffff;
1174 }
1175
1176 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1177 {
1178         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1179         u16 idx = agg_id & MAX_TPA_P5_MASK;
1180
1181         if (test_bit(idx, map->agg_idx_bmap))
1182                 idx = find_first_zero_bit(map->agg_idx_bmap,
1183                                           BNXT_AGG_IDX_BMAP_SIZE);
1184         __set_bit(idx, map->agg_idx_bmap);
1185         map->agg_id_tbl[agg_id] = idx;
1186         return idx;
1187 }
1188
1189 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1190 {
1191         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1192
1193         __clear_bit(idx, map->agg_idx_bmap);
1194 }
1195
1196 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1197 {
1198         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1199
1200         return map->agg_id_tbl[agg_id];
1201 }
1202
1203 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1204                            struct rx_tpa_start_cmp *tpa_start,
1205                            struct rx_tpa_start_cmp_ext *tpa_start1)
1206 {
1207         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1208         struct bnxt_tpa_info *tpa_info;
1209         u16 cons, prod, agg_id;
1210         struct rx_bd *prod_bd;
1211         dma_addr_t mapping;
1212
1213         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1214                 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1215                 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1216         } else {
1217                 agg_id = TPA_START_AGG_ID(tpa_start);
1218         }
1219         cons = tpa_start->rx_tpa_start_cmp_opaque;
1220         prod = rxr->rx_prod;
1221         cons_rx_buf = &rxr->rx_buf_ring[cons];
1222         prod_rx_buf = &rxr->rx_buf_ring[prod];
1223         tpa_info = &rxr->rx_tpa[agg_id];
1224
1225         if (unlikely(cons != rxr->rx_next_cons ||
1226                      TPA_START_ERROR(tpa_start))) {
1227                 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1228                             cons, rxr->rx_next_cons,
1229                             TPA_START_ERROR_CODE(tpa_start1));
1230                 bnxt_sched_reset(bp, rxr);
1231                 return;
1232         }
1233         /* Store cfa_code in tpa_info to use in tpa_end
1234          * completion processing.
1235          */
1236         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1237         prod_rx_buf->data = tpa_info->data;
1238         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1239
1240         mapping = tpa_info->mapping;
1241         prod_rx_buf->mapping = mapping;
1242
1243         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1244
1245         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1246
1247         tpa_info->data = cons_rx_buf->data;
1248         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1249         cons_rx_buf->data = NULL;
1250         tpa_info->mapping = cons_rx_buf->mapping;
1251
1252         tpa_info->len =
1253                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1254                                 RX_TPA_START_CMP_LEN_SHIFT;
1255         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1256                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1257
1258                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1259                 tpa_info->gso_type = SKB_GSO_TCPV4;
1260                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1261                 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1262                         tpa_info->gso_type = SKB_GSO_TCPV6;
1263                 tpa_info->rss_hash =
1264                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1265         } else {
1266                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1267                 tpa_info->gso_type = 0;
1268                 if (netif_msg_rx_err(bp))
1269                         netdev_warn(bp->dev, "TPA packet without valid hash\n");
1270         }
1271         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1272         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1273         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1274         tpa_info->agg_count = 0;
1275
1276         rxr->rx_prod = NEXT_RX(prod);
1277         cons = NEXT_RX(cons);
1278         rxr->rx_next_cons = NEXT_RX(cons);
1279         cons_rx_buf = &rxr->rx_buf_ring[cons];
1280
1281         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1282         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1283         cons_rx_buf->data = NULL;
1284 }
1285
1286 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1287 {
1288         if (agg_bufs)
1289                 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1290 }
1291
1292 #ifdef CONFIG_INET
1293 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1294 {
1295         struct udphdr *uh = NULL;
1296
1297         if (ip_proto == htons(ETH_P_IP)) {
1298                 struct iphdr *iph = (struct iphdr *)skb->data;
1299
1300                 if (iph->protocol == IPPROTO_UDP)
1301                         uh = (struct udphdr *)(iph + 1);
1302         } else {
1303                 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1304
1305                 if (iph->nexthdr == IPPROTO_UDP)
1306                         uh = (struct udphdr *)(iph + 1);
1307         }
1308         if (uh) {
1309                 if (uh->check)
1310                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1311                 else
1312                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1313         }
1314 }
1315 #endif
1316
1317 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1318                                            int payload_off, int tcp_ts,
1319                                            struct sk_buff *skb)
1320 {
1321 #ifdef CONFIG_INET
1322         struct tcphdr *th;
1323         int len, nw_off;
1324         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1325         u32 hdr_info = tpa_info->hdr_info;
1326         bool loopback = false;
1327
1328         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1329         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1330         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1331
1332         /* If the packet is an internal loopback packet, the offsets will
1333          * have an extra 4 bytes.
1334          */
1335         if (inner_mac_off == 4) {
1336                 loopback = true;
1337         } else if (inner_mac_off > 4) {
1338                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1339                                             ETH_HLEN - 2));
1340
1341                 /* We only support inner iPv4/ipv6.  If we don't see the
1342                  * correct protocol ID, it must be a loopback packet where
1343                  * the offsets are off by 4.
1344                  */
1345                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1346                         loopback = true;
1347         }
1348         if (loopback) {
1349                 /* internal loopback packet, subtract all offsets by 4 */
1350                 inner_ip_off -= 4;
1351                 inner_mac_off -= 4;
1352                 outer_ip_off -= 4;
1353         }
1354
1355         nw_off = inner_ip_off - ETH_HLEN;
1356         skb_set_network_header(skb, nw_off);
1357         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1358                 struct ipv6hdr *iph = ipv6_hdr(skb);
1359
1360                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1361                 len = skb->len - skb_transport_offset(skb);
1362                 th = tcp_hdr(skb);
1363                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1364         } else {
1365                 struct iphdr *iph = ip_hdr(skb);
1366
1367                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1368                 len = skb->len - skb_transport_offset(skb);
1369                 th = tcp_hdr(skb);
1370                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1371         }
1372
1373         if (inner_mac_off) { /* tunnel */
1374                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1375                                             ETH_HLEN - 2));
1376
1377                 bnxt_gro_tunnel(skb, proto);
1378         }
1379 #endif
1380         return skb;
1381 }
1382
1383 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1384                                            int payload_off, int tcp_ts,
1385                                            struct sk_buff *skb)
1386 {
1387 #ifdef CONFIG_INET
1388         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1389         u32 hdr_info = tpa_info->hdr_info;
1390         int iphdr_len, nw_off;
1391
1392         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1393         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1394         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1395
1396         nw_off = inner_ip_off - ETH_HLEN;
1397         skb_set_network_header(skb, nw_off);
1398         iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1399                      sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1400         skb_set_transport_header(skb, nw_off + iphdr_len);
1401
1402         if (inner_mac_off) { /* tunnel */
1403                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1404                                             ETH_HLEN - 2));
1405
1406                 bnxt_gro_tunnel(skb, proto);
1407         }
1408 #endif
1409         return skb;
1410 }
1411
1412 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1413 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1414
1415 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1416                                            int payload_off, int tcp_ts,
1417                                            struct sk_buff *skb)
1418 {
1419 #ifdef CONFIG_INET
1420         struct tcphdr *th;
1421         int len, nw_off, tcp_opt_len = 0;
1422
1423         if (tcp_ts)
1424                 tcp_opt_len = 12;
1425
1426         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1427                 struct iphdr *iph;
1428
1429                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1430                          ETH_HLEN;
1431                 skb_set_network_header(skb, nw_off);
1432                 iph = ip_hdr(skb);
1433                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1434                 len = skb->len - skb_transport_offset(skb);
1435                 th = tcp_hdr(skb);
1436                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1437         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1438                 struct ipv6hdr *iph;
1439
1440                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1441                          ETH_HLEN;
1442                 skb_set_network_header(skb, nw_off);
1443                 iph = ipv6_hdr(skb);
1444                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1445                 len = skb->len - skb_transport_offset(skb);
1446                 th = tcp_hdr(skb);
1447                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1448         } else {
1449                 dev_kfree_skb_any(skb);
1450                 return NULL;
1451         }
1452
1453         if (nw_off) /* tunnel */
1454                 bnxt_gro_tunnel(skb, skb->protocol);
1455 #endif
1456         return skb;
1457 }
1458
1459 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1460                                            struct bnxt_tpa_info *tpa_info,
1461                                            struct rx_tpa_end_cmp *tpa_end,
1462                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1463                                            struct sk_buff *skb)
1464 {
1465 #ifdef CONFIG_INET
1466         int payload_off;
1467         u16 segs;
1468
1469         segs = TPA_END_TPA_SEGS(tpa_end);
1470         if (segs == 1)
1471                 return skb;
1472
1473         NAPI_GRO_CB(skb)->count = segs;
1474         skb_shinfo(skb)->gso_size =
1475                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1476         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1477         if (bp->flags & BNXT_FLAG_CHIP_P5)
1478                 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1479         else
1480                 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1481         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1482         if (likely(skb))
1483                 tcp_gro_complete(skb);
1484 #endif
1485         return skb;
1486 }
1487
1488 /* Given the cfa_code of a received packet determine which
1489  * netdev (vf-rep or PF) the packet is destined to.
1490  */
1491 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1492 {
1493         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1494
1495         /* if vf-rep dev is NULL, the must belongs to the PF */
1496         return dev ? dev : bp->dev;
1497 }
1498
1499 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1500                                            struct bnxt_cp_ring_info *cpr,
1501                                            u32 *raw_cons,
1502                                            struct rx_tpa_end_cmp *tpa_end,
1503                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1504                                            u8 *event)
1505 {
1506         struct bnxt_napi *bnapi = cpr->bnapi;
1507         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1508         u8 *data_ptr, agg_bufs;
1509         unsigned int len;
1510         struct bnxt_tpa_info *tpa_info;
1511         dma_addr_t mapping;
1512         struct sk_buff *skb;
1513         u16 idx = 0, agg_id;
1514         void *data;
1515         bool gro;
1516
1517         if (unlikely(bnapi->in_reset)) {
1518                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1519
1520                 if (rc < 0)
1521                         return ERR_PTR(-EBUSY);
1522                 return NULL;
1523         }
1524
1525         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1526                 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1527                 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1528                 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1529                 tpa_info = &rxr->rx_tpa[agg_id];
1530                 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1531                         netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1532                                     agg_bufs, tpa_info->agg_count);
1533                         agg_bufs = tpa_info->agg_count;
1534                 }
1535                 tpa_info->agg_count = 0;
1536                 *event |= BNXT_AGG_EVENT;
1537                 bnxt_free_agg_idx(rxr, agg_id);
1538                 idx = agg_id;
1539                 gro = !!(bp->flags & BNXT_FLAG_GRO);
1540         } else {
1541                 agg_id = TPA_END_AGG_ID(tpa_end);
1542                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1543                 tpa_info = &rxr->rx_tpa[agg_id];
1544                 idx = RING_CMP(*raw_cons);
1545                 if (agg_bufs) {
1546                         if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1547                                 return ERR_PTR(-EBUSY);
1548
1549                         *event |= BNXT_AGG_EVENT;
1550                         idx = NEXT_CMP(idx);
1551                 }
1552                 gro = !!TPA_END_GRO(tpa_end);
1553         }
1554         data = tpa_info->data;
1555         data_ptr = tpa_info->data_ptr;
1556         prefetch(data_ptr);
1557         len = tpa_info->len;
1558         mapping = tpa_info->mapping;
1559
1560         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1561                 bnxt_abort_tpa(cpr, idx, agg_bufs);
1562                 if (agg_bufs > MAX_SKB_FRAGS)
1563                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1564                                     agg_bufs, (int)MAX_SKB_FRAGS);
1565                 return NULL;
1566         }
1567
1568         if (len <= bp->rx_copy_thresh) {
1569                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1570                 if (!skb) {
1571                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1572                         return NULL;
1573                 }
1574         } else {
1575                 u8 *new_data;
1576                 dma_addr_t new_mapping;
1577
1578                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1579                 if (!new_data) {
1580                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1581                         return NULL;
1582                 }
1583
1584                 tpa_info->data = new_data;
1585                 tpa_info->data_ptr = new_data + bp->rx_offset;
1586                 tpa_info->mapping = new_mapping;
1587
1588                 skb = build_skb(data, 0);
1589                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1590                                        bp->rx_buf_use_size, bp->rx_dir,
1591                                        DMA_ATTR_WEAK_ORDERING);
1592
1593                 if (!skb) {
1594                         kfree(data);
1595                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1596                         return NULL;
1597                 }
1598                 skb_reserve(skb, bp->rx_offset);
1599                 skb_put(skb, len);
1600         }
1601
1602         if (agg_bufs) {
1603                 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
1604                 if (!skb) {
1605                         /* Page reuse already handled by bnxt_rx_pages(). */
1606                         return NULL;
1607                 }
1608         }
1609
1610         skb->protocol =
1611                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1612
1613         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1614                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1615
1616         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1617             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1618                 u16 vlan_proto = tpa_info->metadata >>
1619                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1620                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1621
1622                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1623         }
1624
1625         skb_checksum_none_assert(skb);
1626         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1627                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1628                 skb->csum_level =
1629                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1630         }
1631
1632         if (gro)
1633                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1634
1635         return skb;
1636 }
1637
1638 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1639                          struct rx_agg_cmp *rx_agg)
1640 {
1641         u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1642         struct bnxt_tpa_info *tpa_info;
1643
1644         agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1645         tpa_info = &rxr->rx_tpa[agg_id];
1646         BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1647         tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1648 }
1649
1650 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1651                              struct sk_buff *skb)
1652 {
1653         if (skb->dev != bp->dev) {
1654                 /* this packet belongs to a vf-rep */
1655                 bnxt_vf_rep_rx(bp, skb);
1656                 return;
1657         }
1658         skb_record_rx_queue(skb, bnapi->index);
1659         napi_gro_receive(&bnapi->napi, skb);
1660 }
1661
1662 /* returns the following:
1663  * 1       - 1 packet successfully received
1664  * 0       - successful TPA_START, packet not completed yet
1665  * -EBUSY  - completion ring does not have all the agg buffers yet
1666  * -ENOMEM - packet aborted due to out of memory
1667  * -EIO    - packet aborted due to hw error indicated in BD
1668  */
1669 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1670                        u32 *raw_cons, u8 *event)
1671 {
1672         struct bnxt_napi *bnapi = cpr->bnapi;
1673         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1674         struct net_device *dev = bp->dev;
1675         struct rx_cmp *rxcmp;
1676         struct rx_cmp_ext *rxcmp1;
1677         u32 tmp_raw_cons = *raw_cons;
1678         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1679         struct bnxt_sw_rx_bd *rx_buf;
1680         unsigned int len;
1681         u8 *data_ptr, agg_bufs, cmp_type;
1682         dma_addr_t dma_addr;
1683         struct sk_buff *skb;
1684         void *data;
1685         int rc = 0;
1686         u32 misc;
1687
1688         rxcmp = (struct rx_cmp *)
1689                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1690
1691         cmp_type = RX_CMP_TYPE(rxcmp);
1692
1693         if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1694                 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1695                 goto next_rx_no_prod_no_len;
1696         }
1697
1698         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1699         cp_cons = RING_CMP(tmp_raw_cons);
1700         rxcmp1 = (struct rx_cmp_ext *)
1701                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1702
1703         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1704                 return -EBUSY;
1705
1706         prod = rxr->rx_prod;
1707
1708         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1709                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1710                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1711
1712                 *event |= BNXT_RX_EVENT;
1713                 goto next_rx_no_prod_no_len;
1714
1715         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1716                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1717                                    (struct rx_tpa_end_cmp *)rxcmp,
1718                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1719
1720                 if (IS_ERR(skb))
1721                         return -EBUSY;
1722
1723                 rc = -ENOMEM;
1724                 if (likely(skb)) {
1725                         bnxt_deliver_skb(bp, bnapi, skb);
1726                         rc = 1;
1727                 }
1728                 *event |= BNXT_RX_EVENT;
1729                 goto next_rx_no_prod_no_len;
1730         }
1731
1732         cons = rxcmp->rx_cmp_opaque;
1733         if (unlikely(cons != rxr->rx_next_cons)) {
1734                 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
1735
1736                 /* 0xffff is forced error, don't print it */
1737                 if (rxr->rx_next_cons != 0xffff)
1738                         netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1739                                     cons, rxr->rx_next_cons);
1740                 bnxt_sched_reset(bp, rxr);
1741                 return rc1;
1742         }
1743         rx_buf = &rxr->rx_buf_ring[cons];
1744         data = rx_buf->data;
1745         data_ptr = rx_buf->data_ptr;
1746         prefetch(data_ptr);
1747
1748         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1749         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1750
1751         if (agg_bufs) {
1752                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1753                         return -EBUSY;
1754
1755                 cp_cons = NEXT_CMP(cp_cons);
1756                 *event |= BNXT_AGG_EVENT;
1757         }
1758         *event |= BNXT_RX_EVENT;
1759
1760         rx_buf->data = NULL;
1761         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1762                 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1763
1764                 bnxt_reuse_rx_data(rxr, cons, data);
1765                 if (agg_bufs)
1766                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1767                                                false);
1768
1769                 rc = -EIO;
1770                 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1771                         bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1772                         if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
1773                             !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
1774                                 netdev_warn_once(bp->dev, "RX buffer error %x\n",
1775                                                  rx_err);
1776                                 bnxt_sched_reset(bp, rxr);
1777                         }
1778                 }
1779                 goto next_rx_no_len;
1780         }
1781
1782         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1783         dma_addr = rx_buf->mapping;
1784
1785         if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1786                 rc = 1;
1787                 goto next_rx;
1788         }
1789
1790         if (len <= bp->rx_copy_thresh) {
1791                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1792                 bnxt_reuse_rx_data(rxr, cons, data);
1793                 if (!skb) {
1794                         if (agg_bufs)
1795                                 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1796                                                        agg_bufs, false);
1797                         rc = -ENOMEM;
1798                         goto next_rx;
1799                 }
1800         } else {
1801                 u32 payload;
1802
1803                 if (rx_buf->data_ptr == data_ptr)
1804                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1805                 else
1806                         payload = 0;
1807                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1808                                       payload | len);
1809                 if (!skb) {
1810                         rc = -ENOMEM;
1811                         goto next_rx;
1812                 }
1813         }
1814
1815         if (agg_bufs) {
1816                 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
1817                 if (!skb) {
1818                         rc = -ENOMEM;
1819                         goto next_rx;
1820                 }
1821         }
1822
1823         if (RX_CMP_HASH_VALID(rxcmp)) {
1824                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1825                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1826
1827                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1828                 if (hash_type != 1 && hash_type != 3)
1829                         type = PKT_HASH_TYPE_L3;
1830                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1831         }
1832
1833         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1834         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1835
1836         if ((rxcmp1->rx_cmp_flags2 &
1837              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1838             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1839                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1840                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1841                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1842
1843                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1844         }
1845
1846         skb_checksum_none_assert(skb);
1847         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1848                 if (dev->features & NETIF_F_RXCSUM) {
1849                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1850                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1851                 }
1852         } else {
1853                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1854                         if (dev->features & NETIF_F_RXCSUM)
1855                                 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
1856                 }
1857         }
1858
1859         bnxt_deliver_skb(bp, bnapi, skb);
1860         rc = 1;
1861
1862 next_rx:
1863         cpr->rx_packets += 1;
1864         cpr->rx_bytes += len;
1865
1866 next_rx_no_len:
1867         rxr->rx_prod = NEXT_RX(prod);
1868         rxr->rx_next_cons = NEXT_RX(cons);
1869
1870 next_rx_no_prod_no_len:
1871         *raw_cons = tmp_raw_cons;
1872
1873         return rc;
1874 }
1875
1876 /* In netpoll mode, if we are using a combined completion ring, we need to
1877  * discard the rx packets and recycle the buffers.
1878  */
1879 static int bnxt_force_rx_discard(struct bnxt *bp,
1880                                  struct bnxt_cp_ring_info *cpr,
1881                                  u32 *raw_cons, u8 *event)
1882 {
1883         u32 tmp_raw_cons = *raw_cons;
1884         struct rx_cmp_ext *rxcmp1;
1885         struct rx_cmp *rxcmp;
1886         u16 cp_cons;
1887         u8 cmp_type;
1888
1889         cp_cons = RING_CMP(tmp_raw_cons);
1890         rxcmp = (struct rx_cmp *)
1891                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1892
1893         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1894         cp_cons = RING_CMP(tmp_raw_cons);
1895         rxcmp1 = (struct rx_cmp_ext *)
1896                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1897
1898         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1899                 return -EBUSY;
1900
1901         cmp_type = RX_CMP_TYPE(rxcmp);
1902         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1903                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1904                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1905         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1906                 struct rx_tpa_end_cmp_ext *tpa_end1;
1907
1908                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1909                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1910                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1911         }
1912         return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1913 }
1914
1915 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
1916 {
1917         struct bnxt_fw_health *fw_health = bp->fw_health;
1918         u32 reg = fw_health->regs[reg_idx];
1919         u32 reg_type, reg_off, val = 0;
1920
1921         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
1922         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
1923         switch (reg_type) {
1924         case BNXT_FW_HEALTH_REG_TYPE_CFG:
1925                 pci_read_config_dword(bp->pdev, reg_off, &val);
1926                 break;
1927         case BNXT_FW_HEALTH_REG_TYPE_GRC:
1928                 reg_off = fw_health->mapped_regs[reg_idx];
1929                 fallthrough;
1930         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
1931                 val = readl(bp->bar0 + reg_off);
1932                 break;
1933         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
1934                 val = readl(bp->bar1 + reg_off);
1935                 break;
1936         }
1937         if (reg_idx == BNXT_FW_RESET_INPROG_REG)
1938                 val &= fw_health->fw_reset_inprog_reg_mask;
1939         return val;
1940 }
1941
1942 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
1943 {
1944         int i;
1945
1946         for (i = 0; i < bp->rx_nr_rings; i++) {
1947                 u16 grp_idx = bp->rx_ring[i].bnapi->index;
1948                 struct bnxt_ring_grp_info *grp_info;
1949
1950                 grp_info = &bp->grp_info[grp_idx];
1951                 if (grp_info->agg_fw_ring_id == ring_id)
1952                         return grp_idx;
1953         }
1954         return INVALID_HW_RING_ID;
1955 }
1956
1957 #define BNXT_GET_EVENT_PORT(data)       \
1958         ((data) &                       \
1959          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1960
1961 #define BNXT_EVENT_RING_TYPE(data2)     \
1962         ((data2) &                      \
1963          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
1964
1965 #define BNXT_EVENT_RING_TYPE_RX(data2)  \
1966         (BNXT_EVENT_RING_TYPE(data2) == \
1967          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
1968
1969 static int bnxt_async_event_process(struct bnxt *bp,
1970                                     struct hwrm_async_event_cmpl *cmpl)
1971 {
1972         u16 event_id = le16_to_cpu(cmpl->event_id);
1973         u32 data1 = le32_to_cpu(cmpl->event_data1);
1974         u32 data2 = le32_to_cpu(cmpl->event_data2);
1975
1976         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1977         switch (event_id) {
1978         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1979                 struct bnxt_link_info *link_info = &bp->link_info;
1980
1981                 if (BNXT_VF(bp))
1982                         goto async_event_process_exit;
1983
1984                 /* print unsupported speed warning in forced speed mode only */
1985                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1986                     (data1 & 0x20000)) {
1987                         u16 fw_speed = link_info->force_link_speed;
1988                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1989
1990                         if (speed != SPEED_UNKNOWN)
1991                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1992                                             speed);
1993                 }
1994                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1995         }
1996                 fallthrough;
1997         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
1998         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
1999                 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2000                 fallthrough;
2001         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2002                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2003                 break;
2004         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2005                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2006                 break;
2007         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2008                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
2009
2010                 if (BNXT_VF(bp))
2011                         break;
2012
2013                 if (bp->pf.port_id != port_id)
2014                         break;
2015
2016                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2017                 break;
2018         }
2019         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2020                 if (BNXT_PF(bp))
2021                         goto async_event_process_exit;
2022                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2023                 break;
2024         case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY:
2025                 if (netif_msg_hw(bp))
2026                         netdev_warn(bp->dev, "Received RESET_NOTIFY event, data1: 0x%x, data2: 0x%x\n",
2027                                     data1, data2);
2028                 if (!bp->fw_health)
2029                         goto async_event_process_exit;
2030
2031                 bp->fw_reset_timestamp = jiffies;
2032                 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2033                 if (!bp->fw_reset_min_dsecs)
2034                         bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2035                 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2036                 if (!bp->fw_reset_max_dsecs)
2037                         bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2038                 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2039                         netdev_warn(bp->dev, "Firmware fatal reset event received\n");
2040                         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2041                 } else {
2042                         netdev_warn(bp->dev, "Firmware non-fatal reset event received, max wait time %d msec\n",
2043                                     bp->fw_reset_max_dsecs * 100);
2044                 }
2045                 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2046                 break;
2047         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2048                 struct bnxt_fw_health *fw_health = bp->fw_health;
2049
2050                 if (!fw_health)
2051                         goto async_event_process_exit;
2052
2053                 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1);
2054                 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2055                 if (!fw_health->enabled)
2056                         break;
2057
2058                 if (netif_msg_drv(bp))
2059                         netdev_info(bp->dev, "Error recovery info: error recovery[%d], master[%d], reset count[0x%x], health status: 0x%x\n",
2060                                     fw_health->enabled, fw_health->master,
2061                                     bnxt_fw_health_readl(bp,
2062                                                          BNXT_FW_RESET_CNT_REG),
2063                                     bnxt_fw_health_readl(bp,
2064                                                          BNXT_FW_HEALTH_REG));
2065                 fw_health->tmr_multiplier =
2066                         DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2067                                      bp->current_interval * 10);
2068                 fw_health->tmr_counter = fw_health->tmr_multiplier;
2069                 fw_health->last_fw_heartbeat =
2070                         bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2071                 fw_health->last_fw_reset_cnt =
2072                         bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2073                 goto async_event_process_exit;
2074         }
2075         case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2076                 struct bnxt_rx_ring_info *rxr;
2077                 u16 grp_idx;
2078
2079                 if (bp->flags & BNXT_FLAG_CHIP_P5)
2080                         goto async_event_process_exit;
2081
2082                 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2083                             BNXT_EVENT_RING_TYPE(data2), data1);
2084                 if (!BNXT_EVENT_RING_TYPE_RX(data2))
2085                         goto async_event_process_exit;
2086
2087                 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2088                 if (grp_idx == INVALID_HW_RING_ID) {
2089                         netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2090                                     data1);
2091                         goto async_event_process_exit;
2092                 }
2093                 rxr = bp->bnapi[grp_idx]->rx_ring;
2094                 bnxt_sched_reset(bp, rxr);
2095                 goto async_event_process_exit;
2096         }
2097         default:
2098                 goto async_event_process_exit;
2099         }
2100         bnxt_queue_sp_work(bp);
2101 async_event_process_exit:
2102         bnxt_ulp_async_events(bp, cmpl);
2103         return 0;
2104 }
2105
2106 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2107 {
2108         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2109         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2110         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2111                                 (struct hwrm_fwd_req_cmpl *)txcmp;
2112
2113         switch (cmpl_type) {
2114         case CMPL_BASE_TYPE_HWRM_DONE:
2115                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2116                 if (seq_id == bp->hwrm_intr_seq_id)
2117                         bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
2118                 else
2119                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
2120                 break;
2121
2122         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2123                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2124
2125                 if ((vf_id < bp->pf.first_vf_id) ||
2126                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2127                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2128                                    vf_id);
2129                         return -EINVAL;
2130                 }
2131
2132                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2133                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2134                 bnxt_queue_sp_work(bp);
2135                 break;
2136
2137         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2138                 bnxt_async_event_process(bp,
2139                                          (struct hwrm_async_event_cmpl *)txcmp);
2140
2141         default:
2142                 break;
2143         }
2144
2145         return 0;
2146 }
2147
2148 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2149 {
2150         struct bnxt_napi *bnapi = dev_instance;
2151         struct bnxt *bp = bnapi->bp;
2152         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2153         u32 cons = RING_CMP(cpr->cp_raw_cons);
2154
2155         cpr->event_ctr++;
2156         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2157         napi_schedule(&bnapi->napi);
2158         return IRQ_HANDLED;
2159 }
2160
2161 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2162 {
2163         u32 raw_cons = cpr->cp_raw_cons;
2164         u16 cons = RING_CMP(raw_cons);
2165         struct tx_cmp *txcmp;
2166
2167         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2168
2169         return TX_CMP_VALID(txcmp, raw_cons);
2170 }
2171
2172 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2173 {
2174         struct bnxt_napi *bnapi = dev_instance;
2175         struct bnxt *bp = bnapi->bp;
2176         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2177         u32 cons = RING_CMP(cpr->cp_raw_cons);
2178         u32 int_status;
2179
2180         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2181
2182         if (!bnxt_has_work(bp, cpr)) {
2183                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2184                 /* return if erroneous interrupt */
2185                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2186                         return IRQ_NONE;
2187         }
2188
2189         /* disable ring IRQ */
2190         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2191
2192         /* Return here if interrupt is shared and is disabled. */
2193         if (unlikely(atomic_read(&bp->intr_sem) != 0))
2194                 return IRQ_HANDLED;
2195
2196         napi_schedule(&bnapi->napi);
2197         return IRQ_HANDLED;
2198 }
2199
2200 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2201                             int budget)
2202 {
2203         struct bnxt_napi *bnapi = cpr->bnapi;
2204         u32 raw_cons = cpr->cp_raw_cons;
2205         u32 cons;
2206         int tx_pkts = 0;
2207         int rx_pkts = 0;
2208         u8 event = 0;
2209         struct tx_cmp *txcmp;
2210
2211         cpr->has_more_work = 0;
2212         cpr->had_work_done = 1;
2213         while (1) {
2214                 int rc;
2215
2216                 cons = RING_CMP(raw_cons);
2217                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2218
2219                 if (!TX_CMP_VALID(txcmp, raw_cons))
2220                         break;
2221
2222                 /* The valid test of the entry must be done first before
2223                  * reading any further.
2224                  */
2225                 dma_rmb();
2226                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2227                         tx_pkts++;
2228                         /* return full budget so NAPI will complete. */
2229                         if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
2230                                 rx_pkts = budget;
2231                                 raw_cons = NEXT_RAW_CMP(raw_cons);
2232                                 if (budget)
2233                                         cpr->has_more_work = 1;
2234                                 break;
2235                         }
2236                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2237                         if (likely(budget))
2238                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2239                         else
2240                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2241                                                            &event);
2242                         if (likely(rc >= 0))
2243                                 rx_pkts += rc;
2244                         /* Increment rx_pkts when rc is -ENOMEM to count towards
2245                          * the NAPI budget.  Otherwise, we may potentially loop
2246                          * here forever if we consistently cannot allocate
2247                          * buffers.
2248                          */
2249                         else if (rc == -ENOMEM && budget)
2250                                 rx_pkts++;
2251                         else if (rc == -EBUSY)  /* partial completion */
2252                                 break;
2253                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2254                                      CMPL_BASE_TYPE_HWRM_DONE) ||
2255                                     (TX_CMP_TYPE(txcmp) ==
2256                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2257                                     (TX_CMP_TYPE(txcmp) ==
2258                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2259                         bnxt_hwrm_handler(bp, txcmp);
2260                 }
2261                 raw_cons = NEXT_RAW_CMP(raw_cons);
2262
2263                 if (rx_pkts && rx_pkts == budget) {
2264                         cpr->has_more_work = 1;
2265                         break;
2266                 }
2267         }
2268
2269         if (event & BNXT_REDIRECT_EVENT)
2270                 xdp_do_flush_map();
2271
2272         if (event & BNXT_TX_EVENT) {
2273                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2274                 u16 prod = txr->tx_prod;
2275
2276                 /* Sync BD data before updating doorbell */
2277                 wmb();
2278
2279                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2280         }
2281
2282         cpr->cp_raw_cons = raw_cons;
2283         bnapi->tx_pkts += tx_pkts;
2284         bnapi->events |= event;
2285         return rx_pkts;
2286 }
2287
2288 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2289 {
2290         if (bnapi->tx_pkts) {
2291                 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2292                 bnapi->tx_pkts = 0;
2293         }
2294
2295         if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2296                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2297
2298                 if (bnapi->events & BNXT_AGG_EVENT)
2299                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2300                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2301         }
2302         bnapi->events = 0;
2303 }
2304
2305 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2306                           int budget)
2307 {
2308         struct bnxt_napi *bnapi = cpr->bnapi;
2309         int rx_pkts;
2310
2311         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2312
2313         /* ACK completion ring before freeing tx ring and producing new
2314          * buffers in rx/agg rings to prevent overflowing the completion
2315          * ring.
2316          */
2317         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2318
2319         __bnxt_poll_work_done(bp, bnapi);
2320         return rx_pkts;
2321 }
2322
2323 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2324 {
2325         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2326         struct bnxt *bp = bnapi->bp;
2327         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2328         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2329         struct tx_cmp *txcmp;
2330         struct rx_cmp_ext *rxcmp1;
2331         u32 cp_cons, tmp_raw_cons;
2332         u32 raw_cons = cpr->cp_raw_cons;
2333         u32 rx_pkts = 0;
2334         u8 event = 0;
2335
2336         while (1) {
2337                 int rc;
2338
2339                 cp_cons = RING_CMP(raw_cons);
2340                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2341
2342                 if (!TX_CMP_VALID(txcmp, raw_cons))
2343                         break;
2344
2345                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2346                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2347                         cp_cons = RING_CMP(tmp_raw_cons);
2348                         rxcmp1 = (struct rx_cmp_ext *)
2349                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2350
2351                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2352                                 break;
2353
2354                         /* force an error to recycle the buffer */
2355                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2356                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2357
2358                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2359                         if (likely(rc == -EIO) && budget)
2360                                 rx_pkts++;
2361                         else if (rc == -EBUSY)  /* partial completion */
2362                                 break;
2363                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2364                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2365                         bnxt_hwrm_handler(bp, txcmp);
2366                 } else {
2367                         netdev_err(bp->dev,
2368                                    "Invalid completion received on special ring\n");
2369                 }
2370                 raw_cons = NEXT_RAW_CMP(raw_cons);
2371
2372                 if (rx_pkts == budget)
2373                         break;
2374         }
2375
2376         cpr->cp_raw_cons = raw_cons;
2377         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2378         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2379
2380         if (event & BNXT_AGG_EVENT)
2381                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2382
2383         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2384                 napi_complete_done(napi, rx_pkts);
2385                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2386         }
2387         return rx_pkts;
2388 }
2389
2390 static int bnxt_poll(struct napi_struct *napi, int budget)
2391 {
2392         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2393         struct bnxt *bp = bnapi->bp;
2394         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2395         int work_done = 0;
2396
2397         while (1) {
2398                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2399
2400                 if (work_done >= budget) {
2401                         if (!budget)
2402                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2403                         break;
2404                 }
2405
2406                 if (!bnxt_has_work(bp, cpr)) {
2407                         if (napi_complete_done(napi, work_done))
2408                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2409                         break;
2410                 }
2411         }
2412         if (bp->flags & BNXT_FLAG_DIM) {
2413                 struct dim_sample dim_sample = {};
2414
2415                 dim_update_sample(cpr->event_ctr,
2416                                   cpr->rx_packets,
2417                                   cpr->rx_bytes,
2418                                   &dim_sample);
2419                 net_dim(&cpr->dim, dim_sample);
2420         }
2421         return work_done;
2422 }
2423
2424 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2425 {
2426         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2427         int i, work_done = 0;
2428
2429         for (i = 0; i < 2; i++) {
2430                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2431
2432                 if (cpr2) {
2433                         work_done += __bnxt_poll_work(bp, cpr2,
2434                                                       budget - work_done);
2435                         cpr->has_more_work |= cpr2->has_more_work;
2436                 }
2437         }
2438         return work_done;
2439 }
2440
2441 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2442                                  u64 dbr_type)
2443 {
2444         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2445         int i;
2446
2447         for (i = 0; i < 2; i++) {
2448                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2449                 struct bnxt_db_info *db;
2450
2451                 if (cpr2 && cpr2->had_work_done) {
2452                         db = &cpr2->cp_db;
2453                         writeq(db->db_key64 | dbr_type |
2454                                RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2455                         cpr2->had_work_done = 0;
2456                 }
2457         }
2458         __bnxt_poll_work_done(bp, bnapi);
2459 }
2460
2461 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2462 {
2463         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2464         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2465         u32 raw_cons = cpr->cp_raw_cons;
2466         struct bnxt *bp = bnapi->bp;
2467         struct nqe_cn *nqcmp;
2468         int work_done = 0;
2469         u32 cons;
2470
2471         if (cpr->has_more_work) {
2472                 cpr->has_more_work = 0;
2473                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2474         }
2475         while (1) {
2476                 cons = RING_CMP(raw_cons);
2477                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2478
2479                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2480                         if (cpr->has_more_work)
2481                                 break;
2482
2483                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL);
2484                         cpr->cp_raw_cons = raw_cons;
2485                         if (napi_complete_done(napi, work_done))
2486                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2487                                                   cpr->cp_raw_cons);
2488                         return work_done;
2489                 }
2490
2491                 /* The valid test of the entry must be done first before
2492                  * reading any further.
2493                  */
2494                 dma_rmb();
2495
2496                 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2497                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2498                         struct bnxt_cp_ring_info *cpr2;
2499
2500                         cpr2 = cpr->cp_ring_arr[idx];
2501                         work_done += __bnxt_poll_work(bp, cpr2,
2502                                                       budget - work_done);
2503                         cpr->has_more_work |= cpr2->has_more_work;
2504                 } else {
2505                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2506                 }
2507                 raw_cons = NEXT_RAW_CMP(raw_cons);
2508         }
2509         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ);
2510         if (raw_cons != cpr->cp_raw_cons) {
2511                 cpr->cp_raw_cons = raw_cons;
2512                 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2513         }
2514         return work_done;
2515 }
2516
2517 static void bnxt_free_tx_skbs(struct bnxt *bp)
2518 {
2519         int i, max_idx;
2520         struct pci_dev *pdev = bp->pdev;
2521
2522         if (!bp->tx_ring)
2523                 return;
2524
2525         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2526         for (i = 0; i < bp->tx_nr_rings; i++) {
2527                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2528                 int j;
2529
2530                 for (j = 0; j < max_idx;) {
2531                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2532                         struct sk_buff *skb;
2533                         int k, last;
2534
2535                         if (i < bp->tx_nr_rings_xdp &&
2536                             tx_buf->action == XDP_REDIRECT) {
2537                                 dma_unmap_single(&pdev->dev,
2538                                         dma_unmap_addr(tx_buf, mapping),
2539                                         dma_unmap_len(tx_buf, len),
2540                                         PCI_DMA_TODEVICE);
2541                                 xdp_return_frame(tx_buf->xdpf);
2542                                 tx_buf->action = 0;
2543                                 tx_buf->xdpf = NULL;
2544                                 j++;
2545                                 continue;
2546                         }
2547
2548                         skb = tx_buf->skb;
2549                         if (!skb) {
2550                                 j++;
2551                                 continue;
2552                         }
2553
2554                         tx_buf->skb = NULL;
2555
2556                         if (tx_buf->is_push) {
2557                                 dev_kfree_skb(skb);
2558                                 j += 2;
2559                                 continue;
2560                         }
2561
2562                         dma_unmap_single(&pdev->dev,
2563                                          dma_unmap_addr(tx_buf, mapping),
2564                                          skb_headlen(skb),
2565                                          PCI_DMA_TODEVICE);
2566
2567                         last = tx_buf->nr_frags;
2568                         j += 2;
2569                         for (k = 0; k < last; k++, j++) {
2570                                 int ring_idx = j & bp->tx_ring_mask;
2571                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2572
2573                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2574                                 dma_unmap_page(
2575                                         &pdev->dev,
2576                                         dma_unmap_addr(tx_buf, mapping),
2577                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
2578                         }
2579                         dev_kfree_skb(skb);
2580                 }
2581                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2582         }
2583 }
2584
2585 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
2586 {
2587         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
2588         struct pci_dev *pdev = bp->pdev;
2589         struct bnxt_tpa_idx_map *map;
2590         int i, max_idx, max_agg_idx;
2591
2592         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2593         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2594         if (!rxr->rx_tpa)
2595                 goto skip_rx_tpa_free;
2596
2597         for (i = 0; i < bp->max_tpa; i++) {
2598                 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
2599                 u8 *data = tpa_info->data;
2600
2601                 if (!data)
2602                         continue;
2603
2604                 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
2605                                        bp->rx_buf_use_size, bp->rx_dir,
2606                                        DMA_ATTR_WEAK_ORDERING);
2607
2608                 tpa_info->data = NULL;
2609
2610                 kfree(data);
2611         }
2612
2613 skip_rx_tpa_free:
2614         for (i = 0; i < max_idx; i++) {
2615                 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
2616                 dma_addr_t mapping = rx_buf->mapping;
2617                 void *data = rx_buf->data;
2618
2619                 if (!data)
2620                         continue;
2621
2622                 rx_buf->data = NULL;
2623                 if (BNXT_RX_PAGE_MODE(bp)) {
2624                         mapping -= bp->rx_dma_offset;
2625                         dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE,
2626                                              bp->rx_dir,
2627                                              DMA_ATTR_WEAK_ORDERING);
2628                         page_pool_recycle_direct(rxr->page_pool, data);
2629                 } else {
2630                         dma_unmap_single_attrs(&pdev->dev, mapping,
2631                                                bp->rx_buf_use_size, bp->rx_dir,
2632                                                DMA_ATTR_WEAK_ORDERING);
2633                         kfree(data);
2634                 }
2635         }
2636         for (i = 0; i < max_agg_idx; i++) {
2637                 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
2638                 struct page *page = rx_agg_buf->page;
2639
2640                 if (!page)
2641                         continue;
2642
2643                 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2644                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
2645                                      DMA_ATTR_WEAK_ORDERING);
2646
2647                 rx_agg_buf->page = NULL;
2648                 __clear_bit(i, rxr->rx_agg_bmap);
2649
2650                 __free_page(page);
2651         }
2652         if (rxr->rx_page) {
2653                 __free_page(rxr->rx_page);
2654                 rxr->rx_page = NULL;
2655         }
2656         map = rxr->rx_tpa_idx_map;
2657         if (map)
2658                 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2659 }
2660
2661 static void bnxt_free_rx_skbs(struct bnxt *bp)
2662 {
2663         int i;
2664
2665         if (!bp->rx_ring)
2666                 return;
2667
2668         for (i = 0; i < bp->rx_nr_rings; i++)
2669                 bnxt_free_one_rx_ring_skbs(bp, i);
2670 }
2671
2672 static void bnxt_free_skbs(struct bnxt *bp)
2673 {
2674         bnxt_free_tx_skbs(bp);
2675         bnxt_free_rx_skbs(bp);
2676 }
2677
2678 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2679 {
2680         struct pci_dev *pdev = bp->pdev;
2681         int i;
2682
2683         for (i = 0; i < rmem->nr_pages; i++) {
2684                 if (!rmem->pg_arr[i])
2685                         continue;
2686
2687                 dma_free_coherent(&pdev->dev, rmem->page_size,
2688                                   rmem->pg_arr[i], rmem->dma_arr[i]);
2689
2690                 rmem->pg_arr[i] = NULL;
2691         }
2692         if (rmem->pg_tbl) {
2693                 size_t pg_tbl_size = rmem->nr_pages * 8;
2694
2695                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2696                         pg_tbl_size = rmem->page_size;
2697                 dma_free_coherent(&pdev->dev, pg_tbl_size,
2698                                   rmem->pg_tbl, rmem->pg_tbl_map);
2699                 rmem->pg_tbl = NULL;
2700         }
2701         if (rmem->vmem_size && *rmem->vmem) {
2702                 vfree(*rmem->vmem);
2703                 *rmem->vmem = NULL;
2704         }
2705 }
2706
2707 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2708 {
2709         struct pci_dev *pdev = bp->pdev;
2710         u64 valid_bit = 0;
2711         int i;
2712
2713         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2714                 valid_bit = PTU_PTE_VALID;
2715         if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2716                 size_t pg_tbl_size = rmem->nr_pages * 8;
2717
2718                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2719                         pg_tbl_size = rmem->page_size;
2720                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2721                                                   &rmem->pg_tbl_map,
2722                                                   GFP_KERNEL);
2723                 if (!rmem->pg_tbl)
2724                         return -ENOMEM;
2725         }
2726
2727         for (i = 0; i < rmem->nr_pages; i++) {
2728                 u64 extra_bits = valid_bit;
2729
2730                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2731                                                      rmem->page_size,
2732                                                      &rmem->dma_arr[i],
2733                                                      GFP_KERNEL);
2734                 if (!rmem->pg_arr[i])
2735                         return -ENOMEM;
2736
2737                 if (rmem->init_val)
2738                         memset(rmem->pg_arr[i], rmem->init_val,
2739                                rmem->page_size);
2740                 if (rmem->nr_pages > 1 || rmem->depth > 0) {
2741                         if (i == rmem->nr_pages - 2 &&
2742                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2743                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2744                         else if (i == rmem->nr_pages - 1 &&
2745                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2746                                 extra_bits |= PTU_PTE_LAST;
2747                         rmem->pg_tbl[i] =
2748                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2749                 }
2750         }
2751
2752         if (rmem->vmem_size) {
2753                 *rmem->vmem = vzalloc(rmem->vmem_size);
2754                 if (!(*rmem->vmem))
2755                         return -ENOMEM;
2756         }
2757         return 0;
2758 }
2759
2760 static void bnxt_free_tpa_info(struct bnxt *bp)
2761 {
2762         int i;
2763
2764         for (i = 0; i < bp->rx_nr_rings; i++) {
2765                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2766
2767                 kfree(rxr->rx_tpa_idx_map);
2768                 rxr->rx_tpa_idx_map = NULL;
2769                 if (rxr->rx_tpa) {
2770                         kfree(rxr->rx_tpa[0].agg_arr);
2771                         rxr->rx_tpa[0].agg_arr = NULL;
2772                 }
2773                 kfree(rxr->rx_tpa);
2774                 rxr->rx_tpa = NULL;
2775         }
2776 }
2777
2778 static int bnxt_alloc_tpa_info(struct bnxt *bp)
2779 {
2780         int i, j, total_aggs = 0;
2781
2782         bp->max_tpa = MAX_TPA;
2783         if (bp->flags & BNXT_FLAG_CHIP_P5) {
2784                 if (!bp->max_tpa_v2)
2785                         return 0;
2786                 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2787                 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2788         }
2789
2790         for (i = 0; i < bp->rx_nr_rings; i++) {
2791                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2792                 struct rx_agg_cmp *agg;
2793
2794                 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
2795                                       GFP_KERNEL);
2796                 if (!rxr->rx_tpa)
2797                         return -ENOMEM;
2798
2799                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2800                         continue;
2801                 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2802                 rxr->rx_tpa[0].agg_arr = agg;
2803                 if (!agg)
2804                         return -ENOMEM;
2805                 for (j = 1; j < bp->max_tpa; j++)
2806                         rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
2807                 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2808                                               GFP_KERNEL);
2809                 if (!rxr->rx_tpa_idx_map)
2810                         return -ENOMEM;
2811         }
2812         return 0;
2813 }
2814
2815 static void bnxt_free_rx_rings(struct bnxt *bp)
2816 {
2817         int i;
2818
2819         if (!bp->rx_ring)
2820                 return;
2821
2822         bnxt_free_tpa_info(bp);
2823         for (i = 0; i < bp->rx_nr_rings; i++) {
2824                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2825                 struct bnxt_ring_struct *ring;
2826
2827                 if (rxr->xdp_prog)
2828                         bpf_prog_put(rxr->xdp_prog);
2829
2830                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2831                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2832
2833                 page_pool_destroy(rxr->page_pool);
2834                 rxr->page_pool = NULL;
2835
2836                 kfree(rxr->rx_agg_bmap);
2837                 rxr->rx_agg_bmap = NULL;
2838
2839                 ring = &rxr->rx_ring_struct;
2840                 bnxt_free_ring(bp, &ring->ring_mem);
2841
2842                 ring = &rxr->rx_agg_ring_struct;
2843                 bnxt_free_ring(bp, &ring->ring_mem);
2844         }
2845 }
2846
2847 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2848                                    struct bnxt_rx_ring_info *rxr)
2849 {
2850         struct page_pool_params pp = { 0 };
2851
2852         pp.pool_size = bp->rx_ring_size;
2853         pp.nid = dev_to_node(&bp->pdev->dev);
2854         pp.dev = &bp->pdev->dev;
2855         pp.dma_dir = DMA_BIDIRECTIONAL;
2856
2857         rxr->page_pool = page_pool_create(&pp);
2858         if (IS_ERR(rxr->page_pool)) {
2859                 int err = PTR_ERR(rxr->page_pool);
2860
2861                 rxr->page_pool = NULL;
2862                 return err;
2863         }
2864         return 0;
2865 }
2866
2867 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2868 {
2869         int i, rc = 0, agg_rings = 0;
2870
2871         if (!bp->rx_ring)
2872                 return -ENOMEM;
2873
2874         if (bp->flags & BNXT_FLAG_AGG_RINGS)
2875                 agg_rings = 1;
2876
2877         for (i = 0; i < bp->rx_nr_rings; i++) {
2878                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2879                 struct bnxt_ring_struct *ring;
2880
2881                 ring = &rxr->rx_ring_struct;
2882
2883                 rc = bnxt_alloc_rx_page_pool(bp, rxr);
2884                 if (rc)
2885                         return rc;
2886
2887                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
2888                 if (rc < 0)
2889                         return rc;
2890
2891                 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
2892                                                 MEM_TYPE_PAGE_POOL,
2893                                                 rxr->page_pool);
2894                 if (rc) {
2895                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2896                         return rc;
2897                 }
2898
2899                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2900                 if (rc)
2901                         return rc;
2902
2903                 ring->grp_idx = i;
2904                 if (agg_rings) {
2905                         u16 mem_size;
2906
2907                         ring = &rxr->rx_agg_ring_struct;
2908                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2909                         if (rc)
2910                                 return rc;
2911
2912                         ring->grp_idx = i;
2913                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2914                         mem_size = rxr->rx_agg_bmap_size / 8;
2915                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2916                         if (!rxr->rx_agg_bmap)
2917                                 return -ENOMEM;
2918                 }
2919         }
2920         if (bp->flags & BNXT_FLAG_TPA)
2921                 rc = bnxt_alloc_tpa_info(bp);
2922         return rc;
2923 }
2924
2925 static void bnxt_free_tx_rings(struct bnxt *bp)
2926 {
2927         int i;
2928         struct pci_dev *pdev = bp->pdev;
2929
2930         if (!bp->tx_ring)
2931                 return;
2932
2933         for (i = 0; i < bp->tx_nr_rings; i++) {
2934                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2935                 struct bnxt_ring_struct *ring;
2936
2937                 if (txr->tx_push) {
2938                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
2939                                           txr->tx_push, txr->tx_push_mapping);
2940                         txr->tx_push = NULL;
2941                 }
2942
2943                 ring = &txr->tx_ring_struct;
2944
2945                 bnxt_free_ring(bp, &ring->ring_mem);
2946         }
2947 }
2948
2949 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2950 {
2951         int i, j, rc;
2952         struct pci_dev *pdev = bp->pdev;
2953
2954         bp->tx_push_size = 0;
2955         if (bp->tx_push_thresh) {
2956                 int push_size;
2957
2958                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2959                                         bp->tx_push_thresh);
2960
2961                 if (push_size > 256) {
2962                         push_size = 0;
2963                         bp->tx_push_thresh = 0;
2964                 }
2965
2966                 bp->tx_push_size = push_size;
2967         }
2968
2969         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2970                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2971                 struct bnxt_ring_struct *ring;
2972                 u8 qidx;
2973
2974                 ring = &txr->tx_ring_struct;
2975
2976                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2977                 if (rc)
2978                         return rc;
2979
2980                 ring->grp_idx = txr->bnapi->index;
2981                 if (bp->tx_push_size) {
2982                         dma_addr_t mapping;
2983
2984                         /* One pre-allocated DMA buffer to backup
2985                          * TX push operation
2986                          */
2987                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
2988                                                 bp->tx_push_size,
2989                                                 &txr->tx_push_mapping,
2990                                                 GFP_KERNEL);
2991
2992                         if (!txr->tx_push)
2993                                 return -ENOMEM;
2994
2995                         mapping = txr->tx_push_mapping +
2996                                 sizeof(struct tx_push_bd);
2997                         txr->data_mapping = cpu_to_le64(mapping);
2998                 }
2999                 qidx = bp->tc_to_qidx[j];
3000                 ring->queue_id = bp->q_info[qidx].queue_id;
3001                 if (i < bp->tx_nr_rings_xdp)
3002                         continue;
3003                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
3004                         j++;
3005         }
3006         return 0;
3007 }
3008
3009 static void bnxt_free_cp_rings(struct bnxt *bp)
3010 {
3011         int i;
3012
3013         if (!bp->bnapi)
3014                 return;
3015
3016         for (i = 0; i < bp->cp_nr_rings; i++) {
3017                 struct bnxt_napi *bnapi = bp->bnapi[i];
3018                 struct bnxt_cp_ring_info *cpr;
3019                 struct bnxt_ring_struct *ring;
3020                 int j;
3021
3022                 if (!bnapi)
3023                         continue;
3024
3025                 cpr = &bnapi->cp_ring;
3026                 ring = &cpr->cp_ring_struct;
3027
3028                 bnxt_free_ring(bp, &ring->ring_mem);
3029
3030                 for (j = 0; j < 2; j++) {
3031                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3032
3033                         if (cpr2) {
3034                                 ring = &cpr2->cp_ring_struct;
3035                                 bnxt_free_ring(bp, &ring->ring_mem);
3036                                 kfree(cpr2);
3037                                 cpr->cp_ring_arr[j] = NULL;
3038                         }
3039                 }
3040         }
3041 }
3042
3043 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3044 {
3045         struct bnxt_ring_mem_info *rmem;
3046         struct bnxt_ring_struct *ring;
3047         struct bnxt_cp_ring_info *cpr;
3048         int rc;
3049
3050         cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3051         if (!cpr)
3052                 return NULL;
3053
3054         ring = &cpr->cp_ring_struct;
3055         rmem = &ring->ring_mem;
3056         rmem->nr_pages = bp->cp_nr_pages;
3057         rmem->page_size = HW_CMPD_RING_SIZE;
3058         rmem->pg_arr = (void **)cpr->cp_desc_ring;
3059         rmem->dma_arr = cpr->cp_desc_mapping;
3060         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3061         rc = bnxt_alloc_ring(bp, rmem);
3062         if (rc) {
3063                 bnxt_free_ring(bp, rmem);
3064                 kfree(cpr);
3065                 cpr = NULL;
3066         }
3067         return cpr;
3068 }
3069
3070 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3071 {
3072         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3073         int i, rc, ulp_base_vec, ulp_msix;
3074
3075         ulp_msix = bnxt_get_ulp_msix_num(bp);
3076         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3077         for (i = 0; i < bp->cp_nr_rings; i++) {
3078                 struct bnxt_napi *bnapi = bp->bnapi[i];
3079                 struct bnxt_cp_ring_info *cpr;
3080                 struct bnxt_ring_struct *ring;
3081
3082                 if (!bnapi)
3083                         continue;
3084
3085                 cpr = &bnapi->cp_ring;
3086                 cpr->bnapi = bnapi;
3087                 ring = &cpr->cp_ring_struct;
3088
3089                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3090                 if (rc)
3091                         return rc;
3092
3093                 if (ulp_msix && i >= ulp_base_vec)
3094                         ring->map_idx = i + ulp_msix;
3095                 else
3096                         ring->map_idx = i;
3097
3098                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3099                         continue;
3100
3101                 if (i < bp->rx_nr_rings) {
3102                         struct bnxt_cp_ring_info *cpr2 =
3103                                 bnxt_alloc_cp_sub_ring(bp);
3104
3105                         cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3106                         if (!cpr2)
3107                                 return -ENOMEM;
3108                         cpr2->bnapi = bnapi;
3109                 }
3110                 if ((sh && i < bp->tx_nr_rings) ||
3111                     (!sh && i >= bp->rx_nr_rings)) {
3112                         struct bnxt_cp_ring_info *cpr2 =
3113                                 bnxt_alloc_cp_sub_ring(bp);
3114
3115                         cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3116                         if (!cpr2)
3117                                 return -ENOMEM;
3118                         cpr2->bnapi = bnapi;
3119                 }
3120         }
3121         return 0;
3122 }
3123
3124 static void bnxt_init_ring_struct(struct bnxt *bp)
3125 {
3126         int i;
3127
3128         for (i = 0; i < bp->cp_nr_rings; i++) {
3129                 struct bnxt_napi *bnapi = bp->bnapi[i];
3130                 struct bnxt_ring_mem_info *rmem;
3131                 struct bnxt_cp_ring_info *cpr;
3132                 struct bnxt_rx_ring_info *rxr;
3133                 struct bnxt_tx_ring_info *txr;
3134                 struct bnxt_ring_struct *ring;
3135
3136                 if (!bnapi)
3137                         continue;
3138
3139                 cpr = &bnapi->cp_ring;
3140                 ring = &cpr->cp_ring_struct;
3141                 rmem = &ring->ring_mem;
3142                 rmem->nr_pages = bp->cp_nr_pages;
3143                 rmem->page_size = HW_CMPD_RING_SIZE;
3144                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3145                 rmem->dma_arr = cpr->cp_desc_mapping;
3146                 rmem->vmem_size = 0;
3147
3148                 rxr = bnapi->rx_ring;
3149                 if (!rxr)
3150                         goto skip_rx;
3151
3152                 ring = &rxr->rx_ring_struct;
3153                 rmem = &ring->ring_mem;
3154                 rmem->nr_pages = bp->rx_nr_pages;
3155                 rmem->page_size = HW_RXBD_RING_SIZE;
3156                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3157                 rmem->dma_arr = rxr->rx_desc_mapping;
3158                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3159                 rmem->vmem = (void **)&rxr->rx_buf_ring;
3160
3161                 ring = &rxr->rx_agg_ring_struct;
3162                 rmem = &ring->ring_mem;
3163                 rmem->nr_pages = bp->rx_agg_nr_pages;
3164                 rmem->page_size = HW_RXBD_RING_SIZE;
3165                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3166                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3167                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3168                 rmem->vmem = (void **)&rxr->rx_agg_ring;
3169
3170 skip_rx:
3171                 txr = bnapi->tx_ring;
3172                 if (!txr)
3173                         continue;
3174
3175                 ring = &txr->tx_ring_struct;
3176                 rmem = &ring->ring_mem;
3177                 rmem->nr_pages = bp->tx_nr_pages;
3178                 rmem->page_size = HW_RXBD_RING_SIZE;
3179                 rmem->pg_arr = (void **)txr->tx_desc_ring;
3180                 rmem->dma_arr = txr->tx_desc_mapping;
3181                 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3182                 rmem->vmem = (void **)&txr->tx_buf_ring;
3183         }
3184 }
3185
3186 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3187 {
3188         int i;
3189         u32 prod;
3190         struct rx_bd **rx_buf_ring;
3191
3192         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3193         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3194                 int j;
3195                 struct rx_bd *rxbd;
3196
3197                 rxbd = rx_buf_ring[i];
3198                 if (!rxbd)
3199                         continue;
3200
3201                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3202                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3203                         rxbd->rx_bd_opaque = prod;
3204                 }
3205         }
3206 }
3207
3208 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3209 {
3210         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3211         struct net_device *dev = bp->dev;
3212         u32 prod;
3213         int i;
3214
3215         prod = rxr->rx_prod;
3216         for (i = 0; i < bp->rx_ring_size; i++) {
3217                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3218                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3219                                     ring_nr, i, bp->rx_ring_size);
3220                         break;
3221                 }
3222                 prod = NEXT_RX(prod);
3223         }
3224         rxr->rx_prod = prod;
3225
3226         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3227                 return 0;
3228
3229         prod = rxr->rx_agg_prod;
3230         for (i = 0; i < bp->rx_agg_ring_size; i++) {
3231                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
3232                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3233                                     ring_nr, i, bp->rx_ring_size);
3234                         break;
3235                 }
3236                 prod = NEXT_RX_AGG(prod);
3237         }
3238         rxr->rx_agg_prod = prod;
3239
3240         if (rxr->rx_tpa) {
3241                 dma_addr_t mapping;
3242                 u8 *data;
3243
3244                 for (i = 0; i < bp->max_tpa; i++) {
3245                         data = __bnxt_alloc_rx_data(bp, &mapping, GFP_KERNEL);
3246                         if (!data)
3247                                 return -ENOMEM;
3248
3249                         rxr->rx_tpa[i].data = data;
3250                         rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3251                         rxr->rx_tpa[i].mapping = mapping;
3252                 }
3253         }
3254         return 0;
3255 }
3256
3257 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3258 {
3259         struct bnxt_rx_ring_info *rxr;
3260         struct bnxt_ring_struct *ring;
3261         u32 type;
3262
3263         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3264                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3265
3266         if (NET_IP_ALIGN == 2)
3267                 type |= RX_BD_FLAGS_SOP;
3268
3269         rxr = &bp->rx_ring[ring_nr];
3270         ring = &rxr->rx_ring_struct;
3271         bnxt_init_rxbd_pages(ring, type);
3272
3273         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3274                 bpf_prog_add(bp->xdp_prog, 1);
3275                 rxr->xdp_prog = bp->xdp_prog;
3276         }
3277         ring->fw_ring_id = INVALID_HW_RING_ID;
3278
3279         ring = &rxr->rx_agg_ring_struct;
3280         ring->fw_ring_id = INVALID_HW_RING_ID;
3281
3282         if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
3283                 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3284                         RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3285
3286                 bnxt_init_rxbd_pages(ring, type);
3287         }
3288
3289         return bnxt_alloc_one_rx_ring(bp, ring_nr);
3290 }
3291
3292 static void bnxt_init_cp_rings(struct bnxt *bp)
3293 {
3294         int i, j;
3295
3296         for (i = 0; i < bp->cp_nr_rings; i++) {
3297                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3298                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3299
3300                 ring->fw_ring_id = INVALID_HW_RING_ID;
3301                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3302                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3303                 for (j = 0; j < 2; j++) {
3304                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3305
3306                         if (!cpr2)
3307                                 continue;
3308
3309                         ring = &cpr2->cp_ring_struct;
3310                         ring->fw_ring_id = INVALID_HW_RING_ID;
3311                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3312                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3313                 }
3314         }
3315 }
3316
3317 static int bnxt_init_rx_rings(struct bnxt *bp)
3318 {
3319         int i, rc = 0;
3320
3321         if (BNXT_RX_PAGE_MODE(bp)) {
3322                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3323                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3324         } else {
3325                 bp->rx_offset = BNXT_RX_OFFSET;
3326                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3327         }
3328
3329         for (i = 0; i < bp->rx_nr_rings; i++) {
3330                 rc = bnxt_init_one_rx_ring(bp, i);
3331                 if (rc)
3332                         break;
3333         }
3334
3335         return rc;
3336 }
3337
3338 static int bnxt_init_tx_rings(struct bnxt *bp)
3339 {
3340         u16 i;
3341
3342         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3343                                    MAX_SKB_FRAGS + 1);
3344
3345         for (i = 0; i < bp->tx_nr_rings; i++) {
3346                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3347                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3348
3349                 ring->fw_ring_id = INVALID_HW_RING_ID;
3350         }
3351
3352         return 0;
3353 }
3354
3355 static void bnxt_free_ring_grps(struct bnxt *bp)
3356 {
3357         kfree(bp->grp_info);
3358         bp->grp_info = NULL;
3359 }
3360
3361 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3362 {
3363         int i;
3364
3365         if (irq_re_init) {
3366                 bp->grp_info = kcalloc(bp->cp_nr_rings,
3367                                        sizeof(struct bnxt_ring_grp_info),
3368                                        GFP_KERNEL);
3369                 if (!bp->grp_info)
3370                         return -ENOMEM;
3371         }
3372         for (i = 0; i < bp->cp_nr_rings; i++) {
3373                 if (irq_re_init)
3374                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3375                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3376                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3377                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3378                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3379         }
3380         return 0;
3381 }
3382
3383 static void bnxt_free_vnics(struct bnxt *bp)
3384 {
3385         kfree(bp->vnic_info);
3386         bp->vnic_info = NULL;
3387         bp->nr_vnics = 0;
3388 }
3389
3390 static int bnxt_alloc_vnics(struct bnxt *bp)
3391 {
3392         int num_vnics = 1;
3393
3394 #ifdef CONFIG_RFS_ACCEL
3395         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3396                 num_vnics += bp->rx_nr_rings;
3397 #endif
3398
3399         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3400                 num_vnics++;
3401
3402         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3403                                 GFP_KERNEL);
3404         if (!bp->vnic_info)
3405                 return -ENOMEM;
3406
3407         bp->nr_vnics = num_vnics;
3408         return 0;
3409 }
3410
3411 static void bnxt_init_vnics(struct bnxt *bp)
3412 {
3413         int i;
3414
3415         for (i = 0; i < bp->nr_vnics; i++) {
3416                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3417                 int j;
3418
3419                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3420                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3421                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3422
3423                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3424
3425                 if (bp->vnic_info[i].rss_hash_key) {
3426                         if (i == 0)
3427                                 prandom_bytes(vnic->rss_hash_key,
3428                                               HW_HASH_KEY_SIZE);
3429                         else
3430                                 memcpy(vnic->rss_hash_key,
3431                                        bp->vnic_info[0].rss_hash_key,
3432                                        HW_HASH_KEY_SIZE);
3433                 }
3434         }
3435 }
3436
3437 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3438 {
3439         int pages;
3440
3441         pages = ring_size / desc_per_pg;
3442
3443         if (!pages)
3444                 return 1;
3445
3446         pages++;
3447
3448         while (pages & (pages - 1))
3449                 pages++;
3450
3451         return pages;
3452 }
3453
3454 void bnxt_set_tpa_flags(struct bnxt *bp)
3455 {
3456         bp->flags &= ~BNXT_FLAG_TPA;
3457         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3458                 return;
3459         if (bp->dev->features & NETIF_F_LRO)
3460                 bp->flags |= BNXT_FLAG_LRO;
3461         else if (bp->dev->features & NETIF_F_GRO_HW)
3462                 bp->flags |= BNXT_FLAG_GRO;
3463 }
3464
3465 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3466  * be set on entry.
3467  */
3468 void bnxt_set_ring_params(struct bnxt *bp)
3469 {
3470         u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3471         u32 agg_factor = 0, agg_ring_size = 0;
3472
3473         /* 8 for CRC and VLAN */
3474         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3475
3476         rx_space = rx_size + NET_SKB_PAD +
3477                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3478
3479         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3480         ring_size = bp->rx_ring_size;
3481         bp->rx_agg_ring_size = 0;
3482         bp->rx_agg_nr_pages = 0;
3483
3484         if (bp->flags & BNXT_FLAG_TPA)
3485                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3486
3487         bp->flags &= ~BNXT_FLAG_JUMBO;
3488         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3489                 u32 jumbo_factor;
3490
3491                 bp->flags |= BNXT_FLAG_JUMBO;
3492                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3493                 if (jumbo_factor > agg_factor)
3494                         agg_factor = jumbo_factor;
3495         }
3496         agg_ring_size = ring_size * agg_factor;
3497
3498         if (agg_ring_size) {
3499                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3500                                                         RX_DESC_CNT);
3501                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3502                         u32 tmp = agg_ring_size;
3503
3504                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3505                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3506                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3507                                     tmp, agg_ring_size);
3508                 }
3509                 bp->rx_agg_ring_size = agg_ring_size;
3510                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3511                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3512                 rx_space = rx_size + NET_SKB_PAD +
3513                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3514         }
3515
3516         bp->rx_buf_use_size = rx_size;
3517         bp->rx_buf_size = rx_space;
3518
3519         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3520         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3521
3522         ring_size = bp->tx_ring_size;
3523         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3524         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3525
3526         max_rx_cmpl = bp->rx_ring_size;
3527         /* MAX TPA needs to be added because TPA_START completions are
3528          * immediately recycled, so the TPA completions are not bound by
3529          * the RX ring size.
3530          */
3531         if (bp->flags & BNXT_FLAG_TPA)
3532                 max_rx_cmpl += bp->max_tpa;
3533         /* RX and TPA completions are 32-byte, all others are 16-byte */
3534         ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3535         bp->cp_ring_size = ring_size;
3536
3537         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3538         if (bp->cp_nr_pages > MAX_CP_PAGES) {
3539                 bp->cp_nr_pages = MAX_CP_PAGES;
3540                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3541                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3542                             ring_size, bp->cp_ring_size);
3543         }
3544         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3545         bp->cp_ring_mask = bp->cp_bit - 1;
3546 }
3547
3548 /* Changing allocation mode of RX rings.
3549  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3550  */
3551 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3552 {
3553         if (page_mode) {
3554                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3555                         return -EOPNOTSUPP;
3556                 bp->dev->max_mtu =
3557                         min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3558                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3559                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3560                 bp->rx_dir = DMA_BIDIRECTIONAL;
3561                 bp->rx_skb_func = bnxt_rx_page_skb;
3562                 /* Disable LRO or GRO_HW */
3563                 netdev_update_features(bp->dev);
3564         } else {
3565                 bp->dev->max_mtu = bp->max_mtu;
3566                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3567                 bp->rx_dir = DMA_FROM_DEVICE;
3568                 bp->rx_skb_func = bnxt_rx_skb;
3569         }
3570         return 0;
3571 }
3572
3573 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3574 {
3575         int i;
3576         struct bnxt_vnic_info *vnic;
3577         struct pci_dev *pdev = bp->pdev;
3578
3579         if (!bp->vnic_info)
3580                 return;
3581
3582         for (i = 0; i < bp->nr_vnics; i++) {
3583                 vnic = &bp->vnic_info[i];
3584
3585                 kfree(vnic->fw_grp_ids);
3586                 vnic->fw_grp_ids = NULL;
3587
3588                 kfree(vnic->uc_list);
3589                 vnic->uc_list = NULL;
3590
3591                 if (vnic->mc_list) {
3592                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3593                                           vnic->mc_list, vnic->mc_list_mapping);
3594                         vnic->mc_list = NULL;
3595                 }
3596
3597                 if (vnic->rss_table) {
3598                         dma_free_coherent(&pdev->dev, vnic->rss_table_size,
3599                                           vnic->rss_table,
3600                                           vnic->rss_table_dma_addr);
3601                         vnic->rss_table = NULL;
3602                 }
3603
3604                 vnic->rss_hash_key = NULL;
3605                 vnic->flags = 0;
3606         }
3607 }
3608
3609 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3610 {
3611         int i, rc = 0, size;
3612         struct bnxt_vnic_info *vnic;
3613         struct pci_dev *pdev = bp->pdev;
3614         int max_rings;
3615
3616         for (i = 0; i < bp->nr_vnics; i++) {
3617                 vnic = &bp->vnic_info[i];
3618
3619                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3620                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3621
3622                         if (mem_size > 0) {
3623                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3624                                 if (!vnic->uc_list) {
3625                                         rc = -ENOMEM;
3626                                         goto out;
3627                                 }
3628                         }
3629                 }
3630
3631                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3632                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3633                         vnic->mc_list =
3634                                 dma_alloc_coherent(&pdev->dev,
3635                                                    vnic->mc_list_size,
3636                                                    &vnic->mc_list_mapping,
3637                                                    GFP_KERNEL);
3638                         if (!vnic->mc_list) {
3639                                 rc = -ENOMEM;
3640                                 goto out;
3641                         }
3642                 }
3643
3644                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3645                         goto vnic_skip_grps;
3646
3647                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3648                         max_rings = bp->rx_nr_rings;
3649                 else
3650                         max_rings = 1;
3651
3652                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3653                 if (!vnic->fw_grp_ids) {
3654                         rc = -ENOMEM;
3655                         goto out;
3656                 }
3657 vnic_skip_grps:
3658                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3659                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3660                         continue;
3661
3662                 /* Allocate rss table and hash key */
3663                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3664                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3665                         size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
3666
3667                 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
3668                 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
3669                                                      vnic->rss_table_size,
3670                                                      &vnic->rss_table_dma_addr,
3671                                                      GFP_KERNEL);
3672                 if (!vnic->rss_table) {
3673                         rc = -ENOMEM;
3674                         goto out;
3675                 }
3676
3677                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3678                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3679         }
3680         return 0;
3681
3682 out:
3683         return rc;
3684 }
3685
3686 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3687 {
3688         struct pci_dev *pdev = bp->pdev;
3689
3690         if (bp->hwrm_cmd_resp_addr) {
3691                 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3692                                   bp->hwrm_cmd_resp_dma_addr);
3693                 bp->hwrm_cmd_resp_addr = NULL;
3694         }
3695
3696         if (bp->hwrm_cmd_kong_resp_addr) {
3697                 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3698                                   bp->hwrm_cmd_kong_resp_addr,
3699                                   bp->hwrm_cmd_kong_resp_dma_addr);
3700                 bp->hwrm_cmd_kong_resp_addr = NULL;
3701         }
3702 }
3703
3704 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3705 {
3706         struct pci_dev *pdev = bp->pdev;
3707
3708         if (bp->hwrm_cmd_kong_resp_addr)
3709                 return 0;
3710
3711         bp->hwrm_cmd_kong_resp_addr =
3712                 dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3713                                    &bp->hwrm_cmd_kong_resp_dma_addr,
3714                                    GFP_KERNEL);
3715         if (!bp->hwrm_cmd_kong_resp_addr)
3716                 return -ENOMEM;
3717
3718         return 0;
3719 }
3720
3721 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3722 {
3723         struct pci_dev *pdev = bp->pdev;
3724
3725         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3726                                                    &bp->hwrm_cmd_resp_dma_addr,
3727                                                    GFP_KERNEL);
3728         if (!bp->hwrm_cmd_resp_addr)
3729                 return -ENOMEM;
3730
3731         return 0;
3732 }
3733
3734 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3735 {
3736         if (bp->hwrm_short_cmd_req_addr) {
3737                 struct pci_dev *pdev = bp->pdev;
3738
3739                 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3740                                   bp->hwrm_short_cmd_req_addr,
3741                                   bp->hwrm_short_cmd_req_dma_addr);
3742                 bp->hwrm_short_cmd_req_addr = NULL;
3743         }
3744 }
3745
3746 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3747 {
3748         struct pci_dev *pdev = bp->pdev;
3749
3750         if (bp->hwrm_short_cmd_req_addr)
3751                 return 0;
3752
3753         bp->hwrm_short_cmd_req_addr =
3754                 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3755                                    &bp->hwrm_short_cmd_req_dma_addr,
3756                                    GFP_KERNEL);
3757         if (!bp->hwrm_short_cmd_req_addr)
3758                 return -ENOMEM;
3759
3760         return 0;
3761 }
3762
3763 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
3764 {
3765         kfree(stats->hw_masks);
3766         stats->hw_masks = NULL;
3767         kfree(stats->sw_stats);
3768         stats->sw_stats = NULL;
3769         if (stats->hw_stats) {
3770                 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
3771                                   stats->hw_stats_map);
3772                 stats->hw_stats = NULL;
3773         }
3774 }
3775
3776 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
3777                                 bool alloc_masks)
3778 {
3779         stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
3780                                              &stats->hw_stats_map, GFP_KERNEL);
3781         if (!stats->hw_stats)
3782                 return -ENOMEM;
3783
3784         stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
3785         if (!stats->sw_stats)
3786                 goto stats_mem_err;
3787
3788         if (alloc_masks) {
3789                 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
3790                 if (!stats->hw_masks)
3791                         goto stats_mem_err;
3792         }
3793         return 0;
3794
3795 stats_mem_err:
3796         bnxt_free_stats_mem(bp, stats);
3797         return -ENOMEM;
3798 }
3799
3800 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
3801 {
3802         int i;
3803
3804         for (i = 0; i < count; i++)
3805                 mask_arr[i] = mask;
3806 }
3807
3808 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
3809 {
3810         int i;
3811
3812         for (i = 0; i < count; i++)
3813                 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
3814 }
3815
3816 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
3817                                     struct bnxt_stats_mem *stats)
3818 {
3819         struct hwrm_func_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
3820         struct hwrm_func_qstats_ext_input req = {0};
3821         __le64 *hw_masks;
3822         int rc;
3823
3824         if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
3825             !(bp->flags & BNXT_FLAG_CHIP_P5))
3826                 return -EOPNOTSUPP;
3827
3828         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QSTATS_EXT, -1, -1);
3829         req.fid = cpu_to_le16(0xffff);
3830         req.flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3831         mutex_lock(&bp->hwrm_cmd_lock);
3832         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3833         if (rc)
3834                 goto qstat_exit;
3835
3836         hw_masks = &resp->rx_ucast_pkts;
3837         bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
3838
3839 qstat_exit:
3840         mutex_unlock(&bp->hwrm_cmd_lock);
3841         return rc;
3842 }
3843
3844 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
3845 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
3846
3847 static void bnxt_init_stats(struct bnxt *bp)
3848 {
3849         struct bnxt_napi *bnapi = bp->bnapi[0];
3850         struct bnxt_cp_ring_info *cpr;
3851         struct bnxt_stats_mem *stats;
3852         __le64 *rx_stats, *tx_stats;
3853         int rc, rx_count, tx_count;
3854         u64 *rx_masks, *tx_masks;
3855         u64 mask;
3856         u8 flags;
3857
3858         cpr = &bnapi->cp_ring;
3859         stats = &cpr->stats;
3860         rc = bnxt_hwrm_func_qstat_ext(bp, stats);
3861         if (rc) {
3862                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3863                         mask = (1ULL << 48) - 1;
3864                 else
3865                         mask = -1ULL;
3866                 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
3867         }
3868         if (bp->flags & BNXT_FLAG_PORT_STATS) {
3869                 stats = &bp->port_stats;
3870                 rx_stats = stats->hw_stats;
3871                 rx_masks = stats->hw_masks;
3872                 rx_count = sizeof(struct rx_port_stats) / 8;
3873                 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3874                 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3875                 tx_count = sizeof(struct tx_port_stats) / 8;
3876
3877                 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
3878                 rc = bnxt_hwrm_port_qstats(bp, flags);
3879                 if (rc) {
3880                         mask = (1ULL << 40) - 1;
3881
3882                         bnxt_fill_masks(rx_masks, mask, rx_count);
3883                         bnxt_fill_masks(tx_masks, mask, tx_count);
3884                 } else {
3885                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3886                         bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
3887                         bnxt_hwrm_port_qstats(bp, 0);
3888                 }
3889         }
3890         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
3891                 stats = &bp->rx_port_stats_ext;
3892                 rx_stats = stats->hw_stats;
3893                 rx_masks = stats->hw_masks;
3894                 rx_count = sizeof(struct rx_port_stats_ext) / 8;
3895                 stats = &bp->tx_port_stats_ext;
3896                 tx_stats = stats->hw_stats;
3897                 tx_masks = stats->hw_masks;
3898                 tx_count = sizeof(struct tx_port_stats_ext) / 8;
3899
3900                 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3901                 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
3902                 if (rc) {
3903                         mask = (1ULL << 40) - 1;
3904
3905                         bnxt_fill_masks(rx_masks, mask, rx_count);
3906                         if (tx_stats)
3907                                 bnxt_fill_masks(tx_masks, mask, tx_count);
3908                 } else {
3909                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3910                         if (tx_stats)
3911                                 bnxt_copy_hw_masks(tx_masks, tx_stats,
3912                                                    tx_count);
3913                         bnxt_hwrm_port_qstats_ext(bp, 0);
3914                 }
3915         }
3916 }
3917
3918 static void bnxt_free_port_stats(struct bnxt *bp)
3919 {
3920         bp->flags &= ~BNXT_FLAG_PORT_STATS;
3921         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3922
3923         bnxt_free_stats_mem(bp, &bp->port_stats);
3924         bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
3925         bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
3926 }
3927
3928 static void bnxt_free_ring_stats(struct bnxt *bp)
3929 {
3930         int i;
3931
3932         if (!bp->bnapi)
3933                 return;
3934
3935         for (i = 0; i < bp->cp_nr_rings; i++) {
3936                 struct bnxt_napi *bnapi = bp->bnapi[i];
3937                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3938
3939                 bnxt_free_stats_mem(bp, &cpr->stats);
3940         }
3941 }
3942
3943 static int bnxt_alloc_stats(struct bnxt *bp)
3944 {
3945         u32 size, i;
3946         int rc;
3947
3948         size = bp->hw_ring_stats_size;
3949
3950         for (i = 0; i < bp->cp_nr_rings; i++) {
3951                 struct bnxt_napi *bnapi = bp->bnapi[i];
3952                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3953
3954                 cpr->stats.len = size;
3955                 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
3956                 if (rc)
3957                         return rc;
3958
3959                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3960         }
3961
3962         if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
3963                 return 0;
3964
3965         if (bp->port_stats.hw_stats)
3966                 goto alloc_ext_stats;
3967
3968         bp->port_stats.len = BNXT_PORT_STATS_SIZE;
3969         rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
3970         if (rc)
3971                 return rc;
3972
3973         bp->flags |= BNXT_FLAG_PORT_STATS;
3974
3975 alloc_ext_stats:
3976         /* Display extended statistics only if FW supports it */
3977         if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
3978                 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
3979                         return 0;
3980
3981         if (bp->rx_port_stats_ext.hw_stats)
3982                 goto alloc_tx_ext_stats;
3983
3984         bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
3985         rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
3986         /* Extended stats are optional */
3987         if (rc)
3988                 return 0;
3989
3990 alloc_tx_ext_stats:
3991         if (bp->tx_port_stats_ext.hw_stats)
3992                 return 0;
3993
3994         if (bp->hwrm_spec_code >= 0x10902 ||
3995             (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
3996                 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
3997                 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
3998                 /* Extended stats are optional */
3999                 if (rc)
4000                         return 0;
4001         }
4002         bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4003         return 0;
4004 }
4005
4006 static void bnxt_clear_ring_indices(struct bnxt *bp)
4007 {
4008         int i;
4009
4010         if (!bp->bnapi)
4011                 return;
4012
4013         for (i = 0; i < bp->cp_nr_rings; i++) {
4014                 struct bnxt_napi *bnapi = bp->bnapi[i];
4015                 struct bnxt_cp_ring_info *cpr;
4016                 struct bnxt_rx_ring_info *rxr;
4017                 struct bnxt_tx_ring_info *txr;
4018
4019                 if (!bnapi)
4020                         continue;
4021
4022                 cpr = &bnapi->cp_ring;
4023                 cpr->cp_raw_cons = 0;
4024
4025                 txr = bnapi->tx_ring;
4026                 if (txr) {
4027                         txr->tx_prod = 0;
4028                         txr->tx_cons = 0;
4029                 }
4030
4031                 rxr = bnapi->rx_ring;
4032                 if (rxr) {
4033                         rxr->rx_prod = 0;
4034                         rxr->rx_agg_prod = 0;
4035                         rxr->rx_sw_agg_prod = 0;
4036                         rxr->rx_next_cons = 0;
4037                 }
4038         }
4039 }
4040
4041 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
4042 {
4043 #ifdef CONFIG_RFS_ACCEL
4044         int i;
4045
4046         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
4047          * safe to delete the hash table.
4048          */
4049         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4050                 struct hlist_head *head;
4051                 struct hlist_node *tmp;
4052                 struct bnxt_ntuple_filter *fltr;
4053
4054                 head = &bp->ntp_fltr_hash_tbl[i];
4055                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4056                         hlist_del(&fltr->hash);
4057                         kfree(fltr);
4058                 }
4059         }
4060         if (irq_reinit) {
4061                 kfree(bp->ntp_fltr_bmap);
4062                 bp->ntp_fltr_bmap = NULL;
4063         }
4064         bp->ntp_fltr_count = 0;
4065 #endif
4066 }
4067
4068 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4069 {
4070 #ifdef CONFIG_RFS_ACCEL
4071         int i, rc = 0;
4072
4073         if (!(bp->flags & BNXT_FLAG_RFS))
4074                 return 0;
4075
4076         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4077                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4078
4079         bp->ntp_fltr_count = 0;
4080         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
4081                                     sizeof(long),
4082                                     GFP_KERNEL);
4083
4084         if (!bp->ntp_fltr_bmap)
4085                 rc = -ENOMEM;
4086
4087         return rc;
4088 #else
4089         return 0;
4090 #endif
4091 }
4092
4093 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4094 {
4095         bnxt_free_vnic_attributes(bp);
4096         bnxt_free_tx_rings(bp);
4097         bnxt_free_rx_rings(bp);
4098         bnxt_free_cp_rings(bp);
4099         bnxt_free_ntp_fltrs(bp, irq_re_init);
4100         if (irq_re_init) {
4101                 bnxt_free_ring_stats(bp);
4102                 if (!(bp->fw_cap & BNXT_FW_CAP_PORT_STATS_NO_RESET) ||
4103                     test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
4104                         bnxt_free_port_stats(bp);
4105                 bnxt_free_ring_grps(bp);
4106                 bnxt_free_vnics(bp);
4107                 kfree(bp->tx_ring_map);
4108                 bp->tx_ring_map = NULL;
4109                 kfree(bp->tx_ring);
4110                 bp->tx_ring = NULL;
4111                 kfree(bp->rx_ring);
4112                 bp->rx_ring = NULL;
4113                 kfree(bp->bnapi);
4114                 bp->bnapi = NULL;
4115         } else {
4116                 bnxt_clear_ring_indices(bp);
4117         }
4118 }
4119
4120 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4121 {
4122         int i, j, rc, size, arr_size;
4123         void *bnapi;
4124
4125         if (irq_re_init) {
4126                 /* Allocate bnapi mem pointer array and mem block for
4127                  * all queues
4128                  */
4129                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4130                                 bp->cp_nr_rings);
4131                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4132                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4133                 if (!bnapi)
4134                         return -ENOMEM;
4135
4136                 bp->bnapi = bnapi;
4137                 bnapi += arr_size;
4138                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4139                         bp->bnapi[i] = bnapi;
4140                         bp->bnapi[i]->index = i;
4141                         bp->bnapi[i]->bp = bp;
4142                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4143                                 struct bnxt_cp_ring_info *cpr =
4144                                         &bp->bnapi[i]->cp_ring;
4145
4146                                 cpr->cp_ring_struct.ring_mem.flags =
4147                                         BNXT_RMEM_RING_PTE_FLAG;
4148                         }
4149                 }
4150
4151                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
4152                                       sizeof(struct bnxt_rx_ring_info),
4153                                       GFP_KERNEL);
4154                 if (!bp->rx_ring)
4155                         return -ENOMEM;
4156
4157                 for (i = 0; i < bp->rx_nr_rings; i++) {
4158                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4159
4160                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4161                                 rxr->rx_ring_struct.ring_mem.flags =
4162                                         BNXT_RMEM_RING_PTE_FLAG;
4163                                 rxr->rx_agg_ring_struct.ring_mem.flags =
4164                                         BNXT_RMEM_RING_PTE_FLAG;
4165                         }
4166                         rxr->bnapi = bp->bnapi[i];
4167                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4168                 }
4169
4170                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4171                                       sizeof(struct bnxt_tx_ring_info),
4172                                       GFP_KERNEL);
4173                 if (!bp->tx_ring)
4174                         return -ENOMEM;
4175
4176                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4177                                           GFP_KERNEL);
4178
4179                 if (!bp->tx_ring_map)
4180                         return -ENOMEM;
4181
4182                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4183                         j = 0;
4184                 else
4185                         j = bp->rx_nr_rings;
4186
4187                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4188                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4189
4190                         if (bp->flags & BNXT_FLAG_CHIP_P5)
4191                                 txr->tx_ring_struct.ring_mem.flags =
4192                                         BNXT_RMEM_RING_PTE_FLAG;
4193                         txr->bnapi = bp->bnapi[j];
4194                         bp->bnapi[j]->tx_ring = txr;
4195                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4196                         if (i >= bp->tx_nr_rings_xdp) {
4197                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
4198                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
4199                         } else {
4200                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4201                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4202                         }
4203                 }
4204
4205                 rc = bnxt_alloc_stats(bp);
4206                 if (rc)
4207                         goto alloc_mem_err;
4208                 bnxt_init_stats(bp);
4209
4210                 rc = bnxt_alloc_ntp_fltrs(bp);
4211                 if (rc)
4212                         goto alloc_mem_err;
4213
4214                 rc = bnxt_alloc_vnics(bp);
4215                 if (rc)
4216                         goto alloc_mem_err;
4217         }
4218
4219         bnxt_init_ring_struct(bp);
4220
4221         rc = bnxt_alloc_rx_rings(bp);
4222         if (rc)
4223                 goto alloc_mem_err;
4224
4225         rc = bnxt_alloc_tx_rings(bp);
4226         if (rc)
4227                 goto alloc_mem_err;
4228
4229         rc = bnxt_alloc_cp_rings(bp);
4230         if (rc)
4231                 goto alloc_mem_err;
4232
4233         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4234                                   BNXT_VNIC_UCAST_FLAG;
4235         rc = bnxt_alloc_vnic_attributes(bp);
4236         if (rc)
4237                 goto alloc_mem_err;
4238         return 0;
4239
4240 alloc_mem_err:
4241         bnxt_free_mem(bp, true);
4242         return rc;
4243 }
4244
4245 static void bnxt_disable_int(struct bnxt *bp)
4246 {
4247         int i;
4248
4249         if (!bp->bnapi)
4250                 return;
4251
4252         for (i = 0; i < bp->cp_nr_rings; i++) {
4253                 struct bnxt_napi *bnapi = bp->bnapi[i];
4254                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4255                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4256
4257                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
4258                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4259         }
4260 }
4261
4262 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4263 {
4264         struct bnxt_napi *bnapi = bp->bnapi[n];
4265         struct bnxt_cp_ring_info *cpr;
4266
4267         cpr = &bnapi->cp_ring;
4268         return cpr->cp_ring_struct.map_idx;
4269 }
4270
4271 static void bnxt_disable_int_sync(struct bnxt *bp)
4272 {
4273         int i;
4274
4275         atomic_inc(&bp->intr_sem);
4276
4277         bnxt_disable_int(bp);
4278         for (i = 0; i < bp->cp_nr_rings; i++) {
4279                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4280
4281                 synchronize_irq(bp->irq_tbl[map_idx].vector);
4282         }
4283 }
4284
4285 static void bnxt_enable_int(struct bnxt *bp)
4286 {
4287         int i;
4288
4289         atomic_set(&bp->intr_sem, 0);
4290         for (i = 0; i < bp->cp_nr_rings; i++) {
4291                 struct bnxt_napi *bnapi = bp->bnapi[i];
4292                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4293
4294                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4295         }
4296 }
4297
4298 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
4299                             u16 cmpl_ring, u16 target_id)
4300 {
4301         struct input *req = request;
4302
4303         req->req_type = cpu_to_le16(req_type);
4304         req->cmpl_ring = cpu_to_le16(cmpl_ring);
4305         req->target_id = cpu_to_le16(target_id);
4306         if (bnxt_kong_hwrm_message(bp, req))
4307                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
4308         else
4309                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
4310 }
4311
4312 static int bnxt_hwrm_to_stderr(u32 hwrm_err)
4313 {
4314         switch (hwrm_err) {
4315         case HWRM_ERR_CODE_SUCCESS:
4316                 return 0;
4317         case HWRM_ERR_CODE_RESOURCE_LOCKED:
4318                 return -EROFS;
4319         case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
4320                 return -EACCES;
4321         case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
4322                 return -ENOSPC;
4323         case HWRM_ERR_CODE_INVALID_PARAMS:
4324         case HWRM_ERR_CODE_INVALID_FLAGS:
4325         case HWRM_ERR_CODE_INVALID_ENABLES:
4326         case HWRM_ERR_CODE_UNSUPPORTED_TLV:
4327         case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
4328                 return -EINVAL;
4329         case HWRM_ERR_CODE_NO_BUFFER:
4330                 return -ENOMEM;
4331         case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
4332         case HWRM_ERR_CODE_BUSY:
4333                 return -EAGAIN;
4334         case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
4335                 return -EOPNOTSUPP;
4336         default:
4337                 return -EIO;
4338         }
4339 }
4340
4341 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
4342                                  int timeout, bool silent)
4343 {
4344         int i, intr_process, rc, tmo_count;
4345         struct input *req = msg;
4346         u32 *data = msg;
4347         u8 *valid;
4348         u16 cp_ring_id, len = 0;
4349         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
4350         u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
4351         struct hwrm_short_input short_input = {0};
4352         u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
4353         u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
4354         u16 dst = BNXT_HWRM_CHNL_CHIMP;
4355
4356         if (BNXT_NO_FW_ACCESS(bp) &&
4357             le16_to_cpu(req->req_type) != HWRM_FUNC_RESET)
4358                 return -EBUSY;
4359
4360         if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4361                 if (msg_len > bp->hwrm_max_ext_req_len ||
4362                     !bp->hwrm_short_cmd_req_addr)
4363                         return -EINVAL;
4364         }
4365
4366         if (bnxt_hwrm_kong_chnl(bp, req)) {
4367                 dst = BNXT_HWRM_CHNL_KONG;
4368                 bar_offset = BNXT_GRCPF_REG_KONG_COMM;
4369                 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
4370                 resp = bp->hwrm_cmd_kong_resp_addr;
4371         }
4372
4373         memset(resp, 0, PAGE_SIZE);
4374         cp_ring_id = le16_to_cpu(req->cmpl_ring);
4375         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4376
4377         req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4378         /* currently supports only one outstanding message */
4379         if (intr_process)
4380                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4381
4382         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4383             msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4384                 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
4385                 u16 max_msg_len;
4386
4387                 /* Set boundary for maximum extended request length for short
4388                  * cmd format. If passed up from device use the max supported
4389                  * internal req length.
4390                  */
4391                 max_msg_len = bp->hwrm_max_ext_req_len;
4392
4393                 memcpy(short_cmd_req, req, msg_len);
4394                 if (msg_len < max_msg_len)
4395                         memset(short_cmd_req + msg_len, 0,
4396                                max_msg_len - msg_len);
4397
4398                 short_input.req_type = req->req_type;
4399                 short_input.signature =
4400                                 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4401                 short_input.size = cpu_to_le16(msg_len);
4402                 short_input.req_addr =
4403                         cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4404
4405                 data = (u32 *)&short_input;
4406                 msg_len = sizeof(short_input);
4407
4408                 /* Sync memory write before updating doorbell */
4409                 wmb();
4410
4411                 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4412         }
4413
4414         /* Write request msg to hwrm channel */
4415         __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
4416
4417         for (i = msg_len; i < max_req_len; i += 4)
4418                 writel(0, bp->bar0 + bar_offset + i);
4419
4420         /* Ring channel doorbell */
4421         writel(1, bp->bar0 + doorbell_offset);
4422
4423         if (!pci_is_enabled(bp->pdev))
4424                 return 0;
4425
4426         if (!timeout)
4427                 timeout = DFLT_HWRM_CMD_TIMEOUT;
4428         /* convert timeout to usec */
4429         timeout *= 1000;
4430
4431         i = 0;
4432         /* Short timeout for the first few iterations:
4433          * number of loops = number of loops for short timeout +
4434          * number of loops for standard timeout.
4435          */
4436         tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4437         timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4438         tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
4439
4440         if (intr_process) {
4441                 u16 seq_id = bp->hwrm_intr_seq_id;
4442
4443                 /* Wait until hwrm response cmpl interrupt is processed */
4444                 while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
4445                        i++ < tmo_count) {
4446                         /* Abort the wait for completion if the FW health
4447                          * check has failed.
4448                          */
4449                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4450                                 return -EBUSY;
4451                         /* on first few passes, just barely sleep */
4452                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4453                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4454                                              HWRM_SHORT_MAX_TIMEOUT);
4455                         else
4456                                 usleep_range(HWRM_MIN_TIMEOUT,
4457                                              HWRM_MAX_TIMEOUT);
4458                 }
4459
4460                 if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
4461                         if (!silent)
4462                                 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
4463                                            le16_to_cpu(req->req_type));
4464                         return -EBUSY;
4465                 }
4466                 len = le16_to_cpu(resp->resp_len);
4467                 valid = ((u8 *)resp) + len - 1;
4468         } else {
4469                 int j;
4470
4471                 /* Check if response len is updated */
4472                 for (i = 0; i < tmo_count; i++) {
4473                         /* Abort the wait for completion if the FW health
4474                          * check has failed.
4475                          */
4476                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4477                                 return -EBUSY;
4478                         len = le16_to_cpu(resp->resp_len);
4479                         if (len)
4480                                 break;
4481                         /* on first few passes, just barely sleep */
4482                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4483                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4484                                              HWRM_SHORT_MAX_TIMEOUT);
4485                         else
4486                                 usleep_range(HWRM_MIN_TIMEOUT,
4487                                              HWRM_MAX_TIMEOUT);
4488                 }
4489
4490                 if (i >= tmo_count) {
4491                         if (!silent)
4492                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
4493                                            HWRM_TOTAL_TIMEOUT(i),
4494                                            le16_to_cpu(req->req_type),
4495                                            le16_to_cpu(req->seq_id), len);
4496                         return -EBUSY;
4497                 }
4498
4499                 /* Last byte of resp contains valid bit */
4500                 valid = ((u8 *)resp) + len - 1;
4501                 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
4502                         /* make sure we read from updated DMA memory */
4503                         dma_rmb();
4504                         if (*valid)
4505                                 break;
4506                         usleep_range(1, 5);
4507                 }
4508
4509                 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
4510                         if (!silent)
4511                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
4512                                            HWRM_TOTAL_TIMEOUT(i),
4513                                            le16_to_cpu(req->req_type),
4514                                            le16_to_cpu(req->seq_id), len,
4515                                            *valid);
4516                         return -EBUSY;
4517                 }
4518         }
4519
4520         /* Zero valid bit for compatibility.  Valid bit in an older spec
4521          * may become a new field in a newer spec.  We must make sure that
4522          * a new field not implemented by old spec will read zero.
4523          */
4524         *valid = 0;
4525         rc = le16_to_cpu(resp->error_code);
4526         if (rc && !silent)
4527                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4528                            le16_to_cpu(resp->req_type),
4529                            le16_to_cpu(resp->seq_id), rc);
4530         return bnxt_hwrm_to_stderr(rc);
4531 }
4532
4533 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4534 {
4535         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
4536 }
4537
4538 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4539                               int timeout)
4540 {
4541         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4542 }
4543
4544 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4545 {
4546         int rc;
4547
4548         mutex_lock(&bp->hwrm_cmd_lock);
4549         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4550         mutex_unlock(&bp->hwrm_cmd_lock);
4551         return rc;
4552 }
4553
4554 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4555                              int timeout)
4556 {
4557         int rc;
4558
4559         mutex_lock(&bp->hwrm_cmd_lock);
4560         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4561         mutex_unlock(&bp->hwrm_cmd_lock);
4562         return rc;
4563 }
4564
4565 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4566                             bool async_only)
4567 {
4568         struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
4569         struct hwrm_func_drv_rgtr_input req = {0};
4570         DECLARE_BITMAP(async_events_bmap, 256);
4571         u32 *events = (u32 *)async_events_bmap;
4572         u32 flags;
4573         int rc, i;
4574
4575         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4576
4577         req.enables =
4578                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4579                             FUNC_DRV_RGTR_REQ_ENABLES_VER |
4580                             FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4581
4582         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4583         flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4584         if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4585                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4586         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4587                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4588                          FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4589         req.flags = cpu_to_le32(flags);
4590         req.ver_maj_8b = DRV_VER_MAJ;
4591         req.ver_min_8b = DRV_VER_MIN;
4592         req.ver_upd_8b = DRV_VER_UPD;
4593         req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4594         req.ver_min = cpu_to_le16(DRV_VER_MIN);
4595         req.ver_upd = cpu_to_le16(DRV_VER_UPD);
4596
4597         if (BNXT_PF(bp)) {
4598                 u32 data[8];
4599                 int i;
4600
4601                 memset(data, 0, sizeof(data));
4602                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4603                         u16 cmd = bnxt_vf_req_snif[i];
4604                         unsigned int bit, idx;
4605
4606                         idx = cmd / 32;
4607                         bit = cmd % 32;
4608                         data[idx] |= 1 << bit;
4609                 }
4610
4611                 for (i = 0; i < 8; i++)
4612                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4613
4614                 req.enables |=
4615                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4616         }
4617
4618         if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4619                 req.flags |= cpu_to_le32(
4620                         FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4621
4622         memset(async_events_bmap, 0, sizeof(async_events_bmap));
4623         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4624                 u16 event_id = bnxt_async_events_arr[i];
4625
4626                 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4627                     !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4628                         continue;
4629                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4630         }
4631         if (bmap && bmap_size) {
4632                 for (i = 0; i < bmap_size; i++) {
4633                         if (test_bit(i, bmap))
4634                                 __set_bit(i, async_events_bmap);
4635                 }
4636         }
4637         for (i = 0; i < 8; i++)
4638                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4639
4640         if (async_only)
4641                 req.enables =
4642                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4643
4644         mutex_lock(&bp->hwrm_cmd_lock);
4645         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4646         if (!rc) {
4647                 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4648                 if (resp->flags &
4649                     cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4650                         bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4651         }
4652         mutex_unlock(&bp->hwrm_cmd_lock);
4653         return rc;
4654 }
4655
4656 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4657 {
4658         struct hwrm_func_drv_unrgtr_input req = {0};
4659
4660         if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4661                 return 0;
4662
4663         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4664         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4665 }
4666
4667 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4668 {
4669         u32 rc = 0;
4670         struct hwrm_tunnel_dst_port_free_input req = {0};
4671
4672         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4673         req.tunnel_type = tunnel_type;
4674
4675         switch (tunnel_type) {
4676         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4677                 req.tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4678                 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4679                 break;
4680         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4681                 req.tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4682                 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4683                 break;
4684         default:
4685                 break;
4686         }
4687
4688         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4689         if (rc)
4690                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4691                            rc);
4692         return rc;
4693 }
4694
4695 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4696                                            u8 tunnel_type)
4697 {
4698         u32 rc = 0;
4699         struct hwrm_tunnel_dst_port_alloc_input req = {0};
4700         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4701
4702         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4703
4704         req.tunnel_type = tunnel_type;
4705         req.tunnel_dst_port_val = port;
4706
4707         mutex_lock(&bp->hwrm_cmd_lock);
4708         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4709         if (rc) {
4710                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4711                            rc);
4712                 goto err_out;
4713         }
4714
4715         switch (tunnel_type) {
4716         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4717                 bp->vxlan_fw_dst_port_id =
4718                         le16_to_cpu(resp->tunnel_dst_port_id);
4719                 break;
4720         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4721                 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4722                 break;
4723         default:
4724                 break;
4725         }
4726
4727 err_out:
4728         mutex_unlock(&bp->hwrm_cmd_lock);
4729         return rc;
4730 }
4731
4732 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4733 {
4734         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4735         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4736
4737         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4738         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4739
4740         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4741         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4742         req.mask = cpu_to_le32(vnic->rx_mask);
4743         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4744 }
4745
4746 #ifdef CONFIG_RFS_ACCEL
4747 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4748                                             struct bnxt_ntuple_filter *fltr)
4749 {
4750         struct hwrm_cfa_ntuple_filter_free_input req = {0};
4751
4752         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4753         req.ntuple_filter_id = fltr->filter_id;
4754         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4755 }
4756
4757 #define BNXT_NTP_FLTR_FLAGS                                     \
4758         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
4759          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
4760          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
4761          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
4762          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
4763          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
4764          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
4765          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
4766          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
4767          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
4768          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
4769          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
4770          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
4771          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4772
4773 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
4774                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4775
4776 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4777                                              struct bnxt_ntuple_filter *fltr)
4778 {
4779         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4780         struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4781         struct flow_keys *keys = &fltr->fkeys;
4782         struct bnxt_vnic_info *vnic;
4783         u32 flags = 0;
4784         int rc = 0;
4785
4786         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4787         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4788
4789         if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4790                 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4791                 req.dst_id = cpu_to_le16(fltr->rxq);
4792         } else {
4793                 vnic = &bp->vnic_info[fltr->rxq + 1];
4794                 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4795         }
4796         req.flags = cpu_to_le32(flags);
4797         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4798
4799         req.ethertype = htons(ETH_P_IP);
4800         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4801         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4802         req.ip_protocol = keys->basic.ip_proto;
4803
4804         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4805                 int i;
4806
4807                 req.ethertype = htons(ETH_P_IPV6);
4808                 req.ip_addr_type =
4809                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4810                 *(struct in6_addr *)&req.src_ipaddr[0] =
4811                         keys->addrs.v6addrs.src;
4812                 *(struct in6_addr *)&req.dst_ipaddr[0] =
4813                         keys->addrs.v6addrs.dst;
4814                 for (i = 0; i < 4; i++) {
4815                         req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4816                         req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4817                 }
4818         } else {
4819                 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4820                 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4821                 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4822                 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4823         }
4824         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4825                 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4826                 req.tunnel_type =
4827                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4828         }
4829
4830         req.src_port = keys->ports.src;
4831         req.src_port_mask = cpu_to_be16(0xffff);
4832         req.dst_port = keys->ports.dst;
4833         req.dst_port_mask = cpu_to_be16(0xffff);
4834
4835         mutex_lock(&bp->hwrm_cmd_lock);
4836         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4837         if (!rc) {
4838                 resp = bnxt_get_hwrm_resp_addr(bp, &req);
4839                 fltr->filter_id = resp->ntuple_filter_id;
4840         }
4841         mutex_unlock(&bp->hwrm_cmd_lock);
4842         return rc;
4843 }
4844 #endif
4845
4846 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4847                                      u8 *mac_addr)
4848 {
4849         u32 rc = 0;
4850         struct hwrm_cfa_l2_filter_alloc_input req = {0};
4851         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4852
4853         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4854         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4855         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4856                 req.flags |=
4857                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4858         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4859         req.enables =
4860                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4861                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4862                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4863         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4864         req.l2_addr_mask[0] = 0xff;
4865         req.l2_addr_mask[1] = 0xff;
4866         req.l2_addr_mask[2] = 0xff;
4867         req.l2_addr_mask[3] = 0xff;
4868         req.l2_addr_mask[4] = 0xff;
4869         req.l2_addr_mask[5] = 0xff;
4870
4871         mutex_lock(&bp->hwrm_cmd_lock);
4872         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4873         if (!rc)
4874                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4875                                                         resp->l2_filter_id;
4876         mutex_unlock(&bp->hwrm_cmd_lock);
4877         return rc;
4878 }
4879
4880 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4881 {
4882         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4883         int rc = 0;
4884
4885         /* Any associated ntuple filters will also be cleared by firmware. */
4886         mutex_lock(&bp->hwrm_cmd_lock);
4887         for (i = 0; i < num_of_vnics; i++) {
4888                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4889
4890                 for (j = 0; j < vnic->uc_filter_count; j++) {
4891                         struct hwrm_cfa_l2_filter_free_input req = {0};
4892
4893                         bnxt_hwrm_cmd_hdr_init(bp, &req,
4894                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
4895
4896                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
4897
4898                         rc = _hwrm_send_message(bp, &req, sizeof(req),
4899                                                 HWRM_CMD_TIMEOUT);
4900                 }
4901                 vnic->uc_filter_count = 0;
4902         }
4903         mutex_unlock(&bp->hwrm_cmd_lock);
4904
4905         return rc;
4906 }
4907
4908 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4909 {
4910         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4911         u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
4912         struct hwrm_vnic_tpa_cfg_input req = {0};
4913
4914         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4915                 return 0;
4916
4917         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4918
4919         if (tpa_flags) {
4920                 u16 mss = bp->dev->mtu - 40;
4921                 u32 nsegs, n, segs = 0, flags;
4922
4923                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4924                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4925                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4926                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4927                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4928                 if (tpa_flags & BNXT_FLAG_GRO)
4929                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4930
4931                 req.flags = cpu_to_le32(flags);
4932
4933                 req.enables =
4934                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
4935                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4936                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
4937
4938                 /* Number of segs are log2 units, and first packet is not
4939                  * included as part of this units.
4940                  */
4941                 if (mss <= BNXT_RX_PAGE_SIZE) {
4942                         n = BNXT_RX_PAGE_SIZE / mss;
4943                         nsegs = (MAX_SKB_FRAGS - 1) * n;
4944                 } else {
4945                         n = mss / BNXT_RX_PAGE_SIZE;
4946                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
4947                                 n++;
4948                         nsegs = (MAX_SKB_FRAGS - n) / n;
4949                 }
4950
4951                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4952                         segs = MAX_TPA_SEGS_P5;
4953                         max_aggs = bp->max_tpa;
4954                 } else {
4955                         segs = ilog2(nsegs);
4956                 }
4957                 req.max_agg_segs = cpu_to_le16(segs);
4958                 req.max_aggs = cpu_to_le16(max_aggs);
4959
4960                 req.min_agg_len = cpu_to_le32(512);
4961         }
4962         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4963
4964         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4965 }
4966
4967 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4968 {
4969         struct bnxt_ring_grp_info *grp_info;
4970
4971         grp_info = &bp->grp_info[ring->grp_idx];
4972         return grp_info->cp_fw_ring_id;
4973 }
4974
4975 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4976 {
4977         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4978                 struct bnxt_napi *bnapi = rxr->bnapi;
4979                 struct bnxt_cp_ring_info *cpr;
4980
4981                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4982                 return cpr->cp_ring_struct.fw_ring_id;
4983         } else {
4984                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4985         }
4986 }
4987
4988 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4989 {
4990         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4991                 struct bnxt_napi *bnapi = txr->bnapi;
4992                 struct bnxt_cp_ring_info *cpr;
4993
4994                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4995                 return cpr->cp_ring_struct.fw_ring_id;
4996         } else {
4997                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4998         }
4999 }
5000
5001 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
5002 {
5003         int entries;
5004
5005         if (bp->flags & BNXT_FLAG_CHIP_P5)
5006                 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
5007         else
5008                 entries = HW_HASH_INDEX_SIZE;
5009
5010         bp->rss_indir_tbl_entries = entries;
5011         bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
5012                                           GFP_KERNEL);
5013         if (!bp->rss_indir_tbl)
5014                 return -ENOMEM;
5015         return 0;
5016 }
5017
5018 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
5019 {
5020         u16 max_rings, max_entries, pad, i;
5021
5022         if (!bp->rx_nr_rings)
5023                 return;
5024
5025         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5026                 max_rings = bp->rx_nr_rings - 1;
5027         else
5028                 max_rings = bp->rx_nr_rings;
5029
5030         max_entries = bnxt_get_rxfh_indir_size(bp->dev);
5031
5032         for (i = 0; i < max_entries; i++)
5033                 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
5034
5035         pad = bp->rss_indir_tbl_entries - max_entries;
5036         if (pad)
5037                 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
5038 }
5039
5040 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
5041 {
5042         u16 i, tbl_size, max_ring = 0;
5043
5044         if (!bp->rss_indir_tbl)
5045                 return 0;
5046
5047         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5048         for (i = 0; i < tbl_size; i++)
5049                 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5050         return max_ring;
5051 }
5052
5053 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5054 {
5055         if (bp->flags & BNXT_FLAG_CHIP_P5)
5056                 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5057         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5058                 return 2;
5059         return 1;
5060 }
5061
5062 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5063 {
5064         bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5065         u16 i, j;
5066
5067         /* Fill the RSS indirection table with ring group ids */
5068         for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5069                 if (!no_rss)
5070                         j = bp->rss_indir_tbl[i];
5071                 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5072         }
5073 }
5074
5075 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5076                                       struct bnxt_vnic_info *vnic)
5077 {
5078         __le16 *ring_tbl = vnic->rss_table;
5079         struct bnxt_rx_ring_info *rxr;
5080         u16 tbl_size, i;
5081
5082         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5083
5084         for (i = 0; i < tbl_size; i++) {
5085                 u16 ring_id, j;
5086
5087                 j = bp->rss_indir_tbl[i];
5088                 rxr = &bp->rx_ring[j];
5089
5090                 ring_id = rxr->rx_ring_struct.fw_ring_id;
5091                 *ring_tbl++ = cpu_to_le16(ring_id);
5092                 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5093                 *ring_tbl++ = cpu_to_le16(ring_id);
5094         }
5095 }
5096
5097 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5098 {
5099         if (bp->flags & BNXT_FLAG_CHIP_P5)
5100                 __bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5101         else
5102                 __bnxt_fill_hw_rss_tbl(bp, vnic);
5103 }
5104
5105 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5106 {
5107         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5108         struct hwrm_vnic_rss_cfg_input req = {0};
5109
5110         if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5111             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5112                 return 0;
5113
5114         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5115         if (set_rss) {
5116                 bnxt_fill_hw_rss_tbl(bp, vnic);
5117                 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5118                 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5119                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5120                 req.hash_key_tbl_addr =
5121                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
5122         }
5123         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5124         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5125 }
5126
5127 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5128 {
5129         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5130         struct hwrm_vnic_rss_cfg_input req = {0};
5131         dma_addr_t ring_tbl_map;
5132         u32 i, nr_ctxs;
5133
5134         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5135         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5136         if (!set_rss) {
5137                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5138                 return 0;
5139         }
5140         bnxt_fill_hw_rss_tbl(bp, vnic);
5141         req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5142         req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5143         req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5144         ring_tbl_map = vnic->rss_table_dma_addr;
5145         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5146         for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5147                 int rc;
5148
5149                 req.ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5150                 req.ring_table_pair_index = i;
5151                 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5152                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5153                 if (rc)
5154                         return rc;
5155         }
5156         return 0;
5157 }
5158
5159 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5160 {
5161         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5162         struct hwrm_vnic_plcmodes_cfg_input req = {0};
5163
5164         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
5165         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
5166                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5167                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5168         req.enables =
5169                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
5170                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5171         /* thresholds not implemented in firmware yet */
5172         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5173         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5174         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5175         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5176 }
5177
5178 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5179                                         u16 ctx_idx)
5180 {
5181         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
5182
5183         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
5184         req.rss_cos_lb_ctx_id =
5185                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5186
5187         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5188         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5189 }
5190
5191 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5192 {
5193         int i, j;
5194
5195         for (i = 0; i < bp->nr_vnics; i++) {
5196                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5197
5198                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5199                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5200                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5201                 }
5202         }
5203         bp->rsscos_nr_ctxs = 0;
5204 }
5205
5206 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5207 {
5208         int rc;
5209         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
5210         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
5211                                                 bp->hwrm_cmd_resp_addr;
5212
5213         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
5214                                -1);
5215
5216         mutex_lock(&bp->hwrm_cmd_lock);
5217         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5218         if (!rc)
5219                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5220                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
5221         mutex_unlock(&bp->hwrm_cmd_lock);
5222
5223         return rc;
5224 }
5225
5226 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5227 {
5228         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5229                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5230         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5231 }
5232
5233 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5234 {
5235         unsigned int ring = 0, grp_idx;
5236         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5237         struct hwrm_vnic_cfg_input req = {0};
5238         u16 def_vlan = 0;
5239
5240         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
5241
5242         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5243                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5244
5245                 req.default_rx_ring_id =
5246                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5247                 req.default_cmpl_ring_id =
5248                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5249                 req.enables =
5250                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5251                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5252                 goto vnic_mru;
5253         }
5254         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5255         /* Only RSS support for now TBD: COS & LB */
5256         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5257                 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5258                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5259                                            VNIC_CFG_REQ_ENABLES_MRU);
5260         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5261                 req.rss_rule =
5262                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5263                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5264                                            VNIC_CFG_REQ_ENABLES_MRU);
5265                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5266         } else {
5267                 req.rss_rule = cpu_to_le16(0xffff);
5268         }
5269
5270         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5271             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5272                 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5273                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5274         } else {
5275                 req.cos_rule = cpu_to_le16(0xffff);
5276         }
5277
5278         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5279                 ring = 0;
5280         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5281                 ring = vnic_id - 1;
5282         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5283                 ring = bp->rx_nr_rings - 1;
5284
5285         grp_idx = bp->rx_ring[ring].bnapi->index;
5286         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5287         req.lb_rule = cpu_to_le16(0xffff);
5288 vnic_mru:
5289         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5290
5291         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5292 #ifdef CONFIG_BNXT_SRIOV
5293         if (BNXT_VF(bp))
5294                 def_vlan = bp->vf.vlan;
5295 #endif
5296         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5297                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5298         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
5299                 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5300
5301         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5302 }
5303
5304 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5305 {
5306         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5307                 struct hwrm_vnic_free_input req = {0};
5308
5309                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
5310                 req.vnic_id =
5311                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5312
5313                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5314                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5315         }
5316 }
5317
5318 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5319 {
5320         u16 i;
5321
5322         for (i = 0; i < bp->nr_vnics; i++)
5323                 bnxt_hwrm_vnic_free_one(bp, i);
5324 }
5325
5326 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5327                                 unsigned int start_rx_ring_idx,
5328                                 unsigned int nr_rings)
5329 {
5330         int rc = 0;
5331         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5332         struct hwrm_vnic_alloc_input req = {0};
5333         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5334         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5335
5336         if (bp->flags & BNXT_FLAG_CHIP_P5)
5337                 goto vnic_no_ring_grps;
5338
5339         /* map ring groups to this vnic */
5340         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5341                 grp_idx = bp->rx_ring[i].bnapi->index;
5342                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5343                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5344                                    j, nr_rings);
5345                         break;
5346                 }
5347                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5348         }
5349
5350 vnic_no_ring_grps:
5351         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5352                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5353         if (vnic_id == 0)
5354                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5355
5356         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
5357
5358         mutex_lock(&bp->hwrm_cmd_lock);
5359         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5360         if (!rc)
5361                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5362         mutex_unlock(&bp->hwrm_cmd_lock);
5363         return rc;
5364 }
5365
5366 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5367 {
5368         struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5369         struct hwrm_vnic_qcaps_input req = {0};
5370         int rc;
5371
5372         bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5373         bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5374         if (bp->hwrm_spec_code < 0x10600)
5375                 return 0;
5376
5377         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
5378         mutex_lock(&bp->hwrm_cmd_lock);
5379         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5380         if (!rc) {
5381                 u32 flags = le32_to_cpu(resp->flags);
5382
5383                 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5384                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5385                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5386                 if (flags &
5387                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5388                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5389
5390                 /* Older P5 fw before EXT_HW_STATS support did not set
5391                  * VLAN_STRIP_CAP properly.
5392                  */
5393                 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5394                     (BNXT_CHIP_P5_THOR(bp) &&
5395                      !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5396                         bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5397                 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5398                 if (bp->max_tpa_v2) {
5399                         if (BNXT_CHIP_P5_THOR(bp))
5400                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
5401                         else
5402                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2;
5403                 }
5404         }
5405         mutex_unlock(&bp->hwrm_cmd_lock);
5406         return rc;
5407 }
5408
5409 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5410 {
5411         u16 i;
5412         u32 rc = 0;
5413
5414         if (bp->flags & BNXT_FLAG_CHIP_P5)
5415                 return 0;
5416
5417         mutex_lock(&bp->hwrm_cmd_lock);
5418         for (i = 0; i < bp->rx_nr_rings; i++) {
5419                 struct hwrm_ring_grp_alloc_input req = {0};
5420                 struct hwrm_ring_grp_alloc_output *resp =
5421                                         bp->hwrm_cmd_resp_addr;
5422                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5423
5424                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
5425
5426                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5427                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5428                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5429                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5430
5431                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5432                                         HWRM_CMD_TIMEOUT);
5433                 if (rc)
5434                         break;
5435
5436                 bp->grp_info[grp_idx].fw_grp_id =
5437                         le32_to_cpu(resp->ring_group_id);
5438         }
5439         mutex_unlock(&bp->hwrm_cmd_lock);
5440         return rc;
5441 }
5442
5443 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5444 {
5445         u16 i;
5446         struct hwrm_ring_grp_free_input req = {0};
5447
5448         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5449                 return;
5450
5451         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
5452
5453         mutex_lock(&bp->hwrm_cmd_lock);
5454         for (i = 0; i < bp->cp_nr_rings; i++) {
5455                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5456                         continue;
5457                 req.ring_group_id =
5458                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
5459
5460                 _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5461                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5462         }
5463         mutex_unlock(&bp->hwrm_cmd_lock);
5464 }
5465
5466 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5467                                     struct bnxt_ring_struct *ring,
5468                                     u32 ring_type, u32 map_index)
5469 {
5470         int rc = 0, err = 0;
5471         struct hwrm_ring_alloc_input req = {0};
5472         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5473         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5474         struct bnxt_ring_grp_info *grp_info;
5475         u16 ring_id;
5476
5477         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5478
5479         req.enables = 0;
5480         if (rmem->nr_pages > 1) {
5481                 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5482                 /* Page size is in log2 units */
5483                 req.page_size = BNXT_PAGE_SHIFT;
5484                 req.page_tbl_depth = 1;
5485         } else {
5486                 req.page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5487         }
5488         req.fbo = 0;
5489         /* Association of ring index with doorbell index and MSIX number */
5490         req.logical_id = cpu_to_le16(map_index);
5491
5492         switch (ring_type) {
5493         case HWRM_RING_ALLOC_TX: {
5494                 struct bnxt_tx_ring_info *txr;
5495
5496                 txr = container_of(ring, struct bnxt_tx_ring_info,
5497                                    tx_ring_struct);
5498                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5499                 /* Association of transmit ring with completion ring */
5500                 grp_info = &bp->grp_info[ring->grp_idx];
5501                 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5502                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
5503                 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5504                 req.queue_id = cpu_to_le16(ring->queue_id);
5505                 break;
5506         }
5507         case HWRM_RING_ALLOC_RX:
5508                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5509                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
5510                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5511                         u16 flags = 0;
5512
5513                         /* Association of rx ring with stats context */
5514                         grp_info = &bp->grp_info[ring->grp_idx];
5515                         req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5516                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5517                         req.enables |= cpu_to_le32(
5518                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5519                         if (NET_IP_ALIGN == 2)
5520                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5521                         req.flags = cpu_to_le16(flags);
5522                 }
5523                 break;
5524         case HWRM_RING_ALLOC_AGG:
5525                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5526                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5527                         /* Association of agg ring with rx ring */
5528                         grp_info = &bp->grp_info[ring->grp_idx];
5529                         req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5530                         req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5531                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5532                         req.enables |= cpu_to_le32(
5533                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5534                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5535                 } else {
5536                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5537                 }
5538                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5539                 break;
5540         case HWRM_RING_ALLOC_CMPL:
5541                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5542                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5543                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5544                         /* Association of cp ring with nq */
5545                         grp_info = &bp->grp_info[map_index];
5546                         req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5547                         req.cq_handle = cpu_to_le64(ring->handle);
5548                         req.enables |= cpu_to_le32(
5549                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5550                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5551                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5552                 }
5553                 break;
5554         case HWRM_RING_ALLOC_NQ:
5555                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5556                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5557                 if (bp->flags & BNXT_FLAG_USING_MSIX)
5558                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5559                 break;
5560         default:
5561                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5562                            ring_type);
5563                 return -1;
5564         }
5565
5566         mutex_lock(&bp->hwrm_cmd_lock);
5567         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5568         err = le16_to_cpu(resp->error_code);
5569         ring_id = le16_to_cpu(resp->ring_id);
5570         mutex_unlock(&bp->hwrm_cmd_lock);
5571
5572         if (rc || err) {
5573                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5574                            ring_type, rc, err);
5575                 return -EIO;
5576         }
5577         ring->fw_ring_id = ring_id;
5578         return rc;
5579 }
5580
5581 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5582 {
5583         int rc;
5584
5585         if (BNXT_PF(bp)) {
5586                 struct hwrm_func_cfg_input req = {0};
5587
5588                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5589                 req.fid = cpu_to_le16(0xffff);
5590                 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5591                 req.async_event_cr = cpu_to_le16(idx);
5592                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5593         } else {
5594                 struct hwrm_func_vf_cfg_input req = {0};
5595
5596                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5597                 req.enables =
5598                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5599                 req.async_event_cr = cpu_to_le16(idx);
5600                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5601         }
5602         return rc;
5603 }
5604
5605 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5606                         u32 map_idx, u32 xid)
5607 {
5608         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5609                 if (BNXT_PF(bp))
5610                         db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5611                 else
5612                         db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5613                 switch (ring_type) {
5614                 case HWRM_RING_ALLOC_TX:
5615                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5616                         break;
5617                 case HWRM_RING_ALLOC_RX:
5618                 case HWRM_RING_ALLOC_AGG:
5619                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5620                         break;
5621                 case HWRM_RING_ALLOC_CMPL:
5622                         db->db_key64 = DBR_PATH_L2;
5623                         break;
5624                 case HWRM_RING_ALLOC_NQ:
5625                         db->db_key64 = DBR_PATH_L2;
5626                         break;
5627                 }
5628                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5629         } else {
5630                 db->doorbell = bp->bar1 + map_idx * 0x80;
5631                 switch (ring_type) {
5632                 case HWRM_RING_ALLOC_TX:
5633                         db->db_key32 = DB_KEY_TX;
5634                         break;
5635                 case HWRM_RING_ALLOC_RX:
5636                 case HWRM_RING_ALLOC_AGG:
5637                         db->db_key32 = DB_KEY_RX;
5638                         break;
5639                 case HWRM_RING_ALLOC_CMPL:
5640                         db->db_key32 = DB_KEY_CP;
5641                         break;
5642                 }
5643         }
5644 }
5645
5646 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5647 {
5648         bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5649         int i, rc = 0;
5650         u32 type;
5651
5652         if (bp->flags & BNXT_FLAG_CHIP_P5)
5653                 type = HWRM_RING_ALLOC_NQ;
5654         else
5655                 type = HWRM_RING_ALLOC_CMPL;
5656         for (i = 0; i < bp->cp_nr_rings; i++) {
5657                 struct bnxt_napi *bnapi = bp->bnapi[i];
5658                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5659                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5660                 u32 map_idx = ring->map_idx;
5661                 unsigned int vector;
5662
5663                 vector = bp->irq_tbl[map_idx].vector;
5664                 disable_irq_nosync(vector);
5665                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5666                 if (rc) {
5667                         enable_irq(vector);
5668                         goto err_out;
5669                 }
5670                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5671                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5672                 enable_irq(vector);
5673                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5674
5675                 if (!i) {
5676                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5677                         if (rc)
5678                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5679                 }
5680         }
5681
5682         type = HWRM_RING_ALLOC_TX;
5683         for (i = 0; i < bp->tx_nr_rings; i++) {
5684                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5685                 struct bnxt_ring_struct *ring;
5686                 u32 map_idx;
5687
5688                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5689                         struct bnxt_napi *bnapi = txr->bnapi;
5690                         struct bnxt_cp_ring_info *cpr, *cpr2;
5691                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5692
5693                         cpr = &bnapi->cp_ring;
5694                         cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5695                         ring = &cpr2->cp_ring_struct;
5696                         ring->handle = BNXT_TX_HDL;
5697                         map_idx = bnapi->index;
5698                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5699                         if (rc)
5700                                 goto err_out;
5701                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5702                                     ring->fw_ring_id);
5703                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5704                 }
5705                 ring = &txr->tx_ring_struct;
5706                 map_idx = i;
5707                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5708                 if (rc)
5709                         goto err_out;
5710                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5711         }
5712
5713         type = HWRM_RING_ALLOC_RX;
5714         for (i = 0; i < bp->rx_nr_rings; i++) {
5715                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5716                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5717                 struct bnxt_napi *bnapi = rxr->bnapi;
5718                 u32 map_idx = bnapi->index;
5719
5720                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5721                 if (rc)
5722                         goto err_out;
5723                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5724                 /* If we have agg rings, post agg buffers first. */
5725                 if (!agg_rings)
5726                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5727                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5728                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5729                         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5730                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5731                         struct bnxt_cp_ring_info *cpr2;
5732
5733                         cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5734                         ring = &cpr2->cp_ring_struct;
5735                         ring->handle = BNXT_RX_HDL;
5736                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5737                         if (rc)
5738                                 goto err_out;
5739                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5740                                     ring->fw_ring_id);
5741                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5742                 }
5743         }
5744
5745         if (agg_rings) {
5746                 type = HWRM_RING_ALLOC_AGG;
5747                 for (i = 0; i < bp->rx_nr_rings; i++) {
5748                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5749                         struct bnxt_ring_struct *ring =
5750                                                 &rxr->rx_agg_ring_struct;
5751                         u32 grp_idx = ring->grp_idx;
5752                         u32 map_idx = grp_idx + bp->rx_nr_rings;
5753
5754                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5755                         if (rc)
5756                                 goto err_out;
5757
5758                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5759                                     ring->fw_ring_id);
5760                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5761                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5762                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5763                 }
5764         }
5765 err_out:
5766         return rc;
5767 }
5768
5769 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5770                                    struct bnxt_ring_struct *ring,
5771                                    u32 ring_type, int cmpl_ring_id)
5772 {
5773         int rc;
5774         struct hwrm_ring_free_input req = {0};
5775         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5776         u16 error_code;
5777
5778         if (BNXT_NO_FW_ACCESS(bp))
5779                 return 0;
5780
5781         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5782         req.ring_type = ring_type;
5783         req.ring_id = cpu_to_le16(ring->fw_ring_id);
5784
5785         mutex_lock(&bp->hwrm_cmd_lock);
5786         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5787         error_code = le16_to_cpu(resp->error_code);
5788         mutex_unlock(&bp->hwrm_cmd_lock);
5789
5790         if (rc || error_code) {
5791                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5792                            ring_type, rc, error_code);
5793                 return -EIO;
5794         }
5795         return 0;
5796 }
5797
5798 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5799 {
5800         u32 type;
5801         int i;
5802
5803         if (!bp->bnapi)
5804                 return;
5805
5806         for (i = 0; i < bp->tx_nr_rings; i++) {
5807                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5808                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5809
5810                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5811                         u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5812
5813                         hwrm_ring_free_send_msg(bp, ring,
5814                                                 RING_FREE_REQ_RING_TYPE_TX,
5815                                                 close_path ? cmpl_ring_id :
5816                                                 INVALID_HW_RING_ID);
5817                         ring->fw_ring_id = INVALID_HW_RING_ID;
5818                 }
5819         }
5820
5821         for (i = 0; i < bp->rx_nr_rings; i++) {
5822                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5823                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5824                 u32 grp_idx = rxr->bnapi->index;
5825
5826                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5827                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5828
5829                         hwrm_ring_free_send_msg(bp, ring,
5830                                                 RING_FREE_REQ_RING_TYPE_RX,
5831                                                 close_path ? cmpl_ring_id :
5832                                                 INVALID_HW_RING_ID);
5833                         ring->fw_ring_id = INVALID_HW_RING_ID;
5834                         bp->grp_info[grp_idx].rx_fw_ring_id =
5835                                 INVALID_HW_RING_ID;
5836                 }
5837         }
5838
5839         if (bp->flags & BNXT_FLAG_CHIP_P5)
5840                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5841         else
5842                 type = RING_FREE_REQ_RING_TYPE_RX;
5843         for (i = 0; i < bp->rx_nr_rings; i++) {
5844                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5845                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5846                 u32 grp_idx = rxr->bnapi->index;
5847
5848                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5849                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5850
5851                         hwrm_ring_free_send_msg(bp, ring, type,
5852                                                 close_path ? cmpl_ring_id :
5853                                                 INVALID_HW_RING_ID);
5854                         ring->fw_ring_id = INVALID_HW_RING_ID;
5855                         bp->grp_info[grp_idx].agg_fw_ring_id =
5856                                 INVALID_HW_RING_ID;
5857                 }
5858         }
5859
5860         /* The completion rings are about to be freed.  After that the
5861          * IRQ doorbell will not work anymore.  So we need to disable
5862          * IRQ here.
5863          */
5864         bnxt_disable_int_sync(bp);
5865
5866         if (bp->flags & BNXT_FLAG_CHIP_P5)
5867                 type = RING_FREE_REQ_RING_TYPE_NQ;
5868         else
5869                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5870         for (i = 0; i < bp->cp_nr_rings; i++) {
5871                 struct bnxt_napi *bnapi = bp->bnapi[i];
5872                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5873                 struct bnxt_ring_struct *ring;
5874                 int j;
5875
5876                 for (j = 0; j < 2; j++) {
5877                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5878
5879                         if (cpr2) {
5880                                 ring = &cpr2->cp_ring_struct;
5881                                 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5882                                         continue;
5883                                 hwrm_ring_free_send_msg(bp, ring,
5884                                         RING_FREE_REQ_RING_TYPE_L2_CMPL,
5885                                         INVALID_HW_RING_ID);
5886                                 ring->fw_ring_id = INVALID_HW_RING_ID;
5887                         }
5888                 }
5889                 ring = &cpr->cp_ring_struct;
5890                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5891                         hwrm_ring_free_send_msg(bp, ring, type,
5892                                                 INVALID_HW_RING_ID);
5893                         ring->fw_ring_id = INVALID_HW_RING_ID;
5894                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5895                 }
5896         }
5897 }
5898
5899 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5900                            bool shared);
5901
5902 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5903 {
5904         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5905         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5906         struct hwrm_func_qcfg_input req = {0};
5907         int rc;
5908
5909         if (bp->hwrm_spec_code < 0x10601)
5910                 return 0;
5911
5912         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5913         req.fid = cpu_to_le16(0xffff);
5914         mutex_lock(&bp->hwrm_cmd_lock);
5915         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5916         if (rc) {
5917                 mutex_unlock(&bp->hwrm_cmd_lock);
5918                 return rc;
5919         }
5920
5921         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5922         if (BNXT_NEW_RM(bp)) {
5923                 u16 cp, stats;
5924
5925                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5926                 hw_resc->resv_hw_ring_grps =
5927                         le32_to_cpu(resp->alloc_hw_ring_grps);
5928                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5929                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
5930                 stats = le16_to_cpu(resp->alloc_stat_ctx);
5931                 hw_resc->resv_irqs = cp;
5932                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5933                         int rx = hw_resc->resv_rx_rings;
5934                         int tx = hw_resc->resv_tx_rings;
5935
5936                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5937                                 rx >>= 1;
5938                         if (cp < (rx + tx)) {
5939                                 bnxt_trim_rings(bp, &rx, &tx, cp, false);
5940                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5941                                         rx <<= 1;
5942                                 hw_resc->resv_rx_rings = rx;
5943                                 hw_resc->resv_tx_rings = tx;
5944                         }
5945                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
5946                         hw_resc->resv_hw_ring_grps = rx;
5947                 }
5948                 hw_resc->resv_cp_rings = cp;
5949                 hw_resc->resv_stat_ctxs = stats;
5950         }
5951         mutex_unlock(&bp->hwrm_cmd_lock);
5952         return 0;
5953 }
5954
5955 /* Caller must hold bp->hwrm_cmd_lock */
5956 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5957 {
5958         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5959         struct hwrm_func_qcfg_input req = {0};
5960         int rc;
5961
5962         if (bp->hwrm_spec_code < 0x10601)
5963                 return 0;
5964
5965         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5966         req.fid = cpu_to_le16(fid);
5967         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5968         if (!rc)
5969                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5970
5971         return rc;
5972 }
5973
5974 static bool bnxt_rfs_supported(struct bnxt *bp);
5975
5976 static void
5977 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5978                              int tx_rings, int rx_rings, int ring_grps,
5979                              int cp_rings, int stats, int vnics)
5980 {
5981         u32 enables = 0;
5982
5983         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5984         req->fid = cpu_to_le16(0xffff);
5985         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5986         req->num_tx_rings = cpu_to_le16(tx_rings);
5987         if (BNXT_NEW_RM(bp)) {
5988                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
5989                 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5990                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5991                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5992                         enables |= tx_rings + ring_grps ?
5993                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5994                         enables |= rx_rings ?
5995                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5996                 } else {
5997                         enables |= cp_rings ?
5998                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5999                         enables |= ring_grps ?
6000                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
6001                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6002                 }
6003                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
6004
6005                 req->num_rx_rings = cpu_to_le16(rx_rings);
6006                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6007                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6008                         req->num_msix = cpu_to_le16(cp_rings);
6009                         req->num_rsscos_ctxs =
6010                                 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6011                 } else {
6012                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
6013                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6014                         req->num_rsscos_ctxs = cpu_to_le16(1);
6015                         if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
6016                             bnxt_rfs_supported(bp))
6017                                 req->num_rsscos_ctxs =
6018                                         cpu_to_le16(ring_grps + 1);
6019                 }
6020                 req->num_stat_ctxs = cpu_to_le16(stats);
6021                 req->num_vnics = cpu_to_le16(vnics);
6022         }
6023         req->enables = cpu_to_le32(enables);
6024 }
6025
6026 static void
6027 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
6028                              struct hwrm_func_vf_cfg_input *req, int tx_rings,
6029                              int rx_rings, int ring_grps, int cp_rings,
6030                              int stats, int vnics)
6031 {
6032         u32 enables = 0;
6033
6034         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
6035         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6036         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
6037                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6038         enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6039         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6040                 enables |= tx_rings + ring_grps ?
6041                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6042         } else {
6043                 enables |= cp_rings ?
6044                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6045                 enables |= ring_grps ?
6046                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
6047         }
6048         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
6049         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
6050
6051         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6052         req->num_tx_rings = cpu_to_le16(tx_rings);
6053         req->num_rx_rings = cpu_to_le16(rx_rings);
6054         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6055                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6056                 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6057         } else {
6058                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
6059                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6060                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6061         }
6062         req->num_stat_ctxs = cpu_to_le16(stats);
6063         req->num_vnics = cpu_to_le16(vnics);
6064
6065         req->enables = cpu_to_le32(enables);
6066 }
6067
6068 static int
6069 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6070                            int ring_grps, int cp_rings, int stats, int vnics)
6071 {
6072         struct hwrm_func_cfg_input req = {0};
6073         int rc;
6074
6075         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6076                                      cp_rings, stats, vnics);
6077         if (!req.enables)
6078                 return 0;
6079
6080         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6081         if (rc)
6082                 return rc;
6083
6084         if (bp->hwrm_spec_code < 0x10601)
6085                 bp->hw_resc.resv_tx_rings = tx_rings;
6086
6087         return bnxt_hwrm_get_rings(bp);
6088 }
6089
6090 static int
6091 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6092                            int ring_grps, int cp_rings, int stats, int vnics)
6093 {
6094         struct hwrm_func_vf_cfg_input req = {0};
6095         int rc;
6096
6097         if (!BNXT_NEW_RM(bp)) {
6098                 bp->hw_resc.resv_tx_rings = tx_rings;
6099                 return 0;
6100         }
6101
6102         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6103                                      cp_rings, stats, vnics);
6104         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6105         if (rc)
6106                 return rc;
6107
6108         return bnxt_hwrm_get_rings(bp);
6109 }
6110
6111 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6112                                    int cp, int stat, int vnic)
6113 {
6114         if (BNXT_PF(bp))
6115                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6116                                                   vnic);
6117         else
6118                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6119                                                   vnic);
6120 }
6121
6122 int bnxt_nq_rings_in_use(struct bnxt *bp)
6123 {
6124         int cp = bp->cp_nr_rings;
6125         int ulp_msix, ulp_base;
6126
6127         ulp_msix = bnxt_get_ulp_msix_num(bp);
6128         if (ulp_msix) {
6129                 ulp_base = bnxt_get_ulp_msix_base(bp);
6130                 cp += ulp_msix;
6131                 if ((ulp_base + ulp_msix) > cp)
6132                         cp = ulp_base + ulp_msix;
6133         }
6134         return cp;
6135 }
6136
6137 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6138 {
6139         int cp;
6140
6141         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6142                 return bnxt_nq_rings_in_use(bp);
6143
6144         cp = bp->tx_nr_rings + bp->rx_nr_rings;
6145         return cp;
6146 }
6147
6148 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6149 {
6150         int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6151         int cp = bp->cp_nr_rings;
6152
6153         if (!ulp_stat)
6154                 return cp;
6155
6156         if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6157                 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6158
6159         return cp + ulp_stat;
6160 }
6161
6162 /* Check if a default RSS map needs to be setup.  This function is only
6163  * used on older firmware that does not require reserving RX rings.
6164  */
6165 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6166 {
6167         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6168
6169         /* The RSS map is valid for RX rings set to resv_rx_rings */
6170         if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6171                 hw_resc->resv_rx_rings = bp->rx_nr_rings;
6172                 if (!netif_is_rxfh_configured(bp->dev))
6173                         bnxt_set_dflt_rss_indir_tbl(bp);
6174         }
6175 }
6176
6177 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6178 {
6179         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6180         int cp = bnxt_cp_rings_in_use(bp);
6181         int nq = bnxt_nq_rings_in_use(bp);
6182         int rx = bp->rx_nr_rings, stat;
6183         int vnic = 1, grp = rx;
6184
6185         if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6186             bp->hwrm_spec_code >= 0x10601)
6187                 return true;
6188
6189         /* Old firmware does not need RX ring reservations but we still
6190          * need to setup a default RSS map when needed.  With new firmware
6191          * we go through RX ring reservations first and then set up the
6192          * RSS map for the successfully reserved RX rings when needed.
6193          */
6194         if (!BNXT_NEW_RM(bp)) {
6195                 bnxt_check_rss_tbl_no_rmgr(bp);
6196                 return false;
6197         }
6198         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6199                 vnic = rx + 1;
6200         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6201                 rx <<= 1;
6202         stat = bnxt_get_func_stat_ctxs(bp);
6203         if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6204             hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6205             (hw_resc->resv_hw_ring_grps != grp &&
6206              !(bp->flags & BNXT_FLAG_CHIP_P5)))
6207                 return true;
6208         if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6209             hw_resc->resv_irqs != nq)
6210                 return true;
6211         return false;
6212 }
6213
6214 static int __bnxt_reserve_rings(struct bnxt *bp)
6215 {
6216         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6217         int cp = bnxt_nq_rings_in_use(bp);
6218         int tx = bp->tx_nr_rings;
6219         int rx = bp->rx_nr_rings;
6220         int grp, rx_rings, rc;
6221         int vnic = 1, stat;
6222         bool sh = false;
6223
6224         if (!bnxt_need_reserve_rings(bp))
6225                 return 0;
6226
6227         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6228                 sh = true;
6229         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6230                 vnic = rx + 1;
6231         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6232                 rx <<= 1;
6233         grp = bp->rx_nr_rings;
6234         stat = bnxt_get_func_stat_ctxs(bp);
6235
6236         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6237         if (rc)
6238                 return rc;
6239
6240         tx = hw_resc->resv_tx_rings;
6241         if (BNXT_NEW_RM(bp)) {
6242                 rx = hw_resc->resv_rx_rings;
6243                 cp = hw_resc->resv_irqs;
6244                 grp = hw_resc->resv_hw_ring_grps;
6245                 vnic = hw_resc->resv_vnics;
6246                 stat = hw_resc->resv_stat_ctxs;
6247         }
6248
6249         rx_rings = rx;
6250         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6251                 if (rx >= 2) {
6252                         rx_rings = rx >> 1;
6253                 } else {
6254                         if (netif_running(bp->dev))
6255                                 return -ENOMEM;
6256
6257                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6258                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6259                         bp->dev->hw_features &= ~NETIF_F_LRO;
6260                         bp->dev->features &= ~NETIF_F_LRO;
6261                         bnxt_set_ring_params(bp);
6262                 }
6263         }
6264         rx_rings = min_t(int, rx_rings, grp);
6265         cp = min_t(int, cp, bp->cp_nr_rings);
6266         if (stat > bnxt_get_ulp_stat_ctxs(bp))
6267                 stat -= bnxt_get_ulp_stat_ctxs(bp);
6268         cp = min_t(int, cp, stat);
6269         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6270         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6271                 rx = rx_rings << 1;
6272         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6273         bp->tx_nr_rings = tx;
6274
6275         /* If we cannot reserve all the RX rings, reset the RSS map only
6276          * if absolutely necessary
6277          */
6278         if (rx_rings != bp->rx_nr_rings) {
6279                 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6280                             rx_rings, bp->rx_nr_rings);
6281                 if ((bp->dev->priv_flags & IFF_RXFH_CONFIGURED) &&
6282                     (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6283                      bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6284                      bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6285                         netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6286                         bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6287                 }
6288         }
6289         bp->rx_nr_rings = rx_rings;
6290         bp->cp_nr_rings = cp;
6291
6292         if (!tx || !rx || !cp || !grp || !vnic || !stat)
6293                 return -ENOMEM;
6294
6295         if (!netif_is_rxfh_configured(bp->dev))
6296                 bnxt_set_dflt_rss_indir_tbl(bp);
6297
6298         return rc;
6299 }
6300
6301 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6302                                     int ring_grps, int cp_rings, int stats,
6303                                     int vnics)
6304 {
6305         struct hwrm_func_vf_cfg_input req = {0};
6306         u32 flags;
6307
6308         if (!BNXT_NEW_RM(bp))
6309                 return 0;
6310
6311         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6312                                      cp_rings, stats, vnics);
6313         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6314                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6315                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6316                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6317                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6318                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6319         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6320                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6321
6322         req.flags = cpu_to_le32(flags);
6323         return hwrm_send_message_silent(bp, &req, sizeof(req),
6324                                         HWRM_CMD_TIMEOUT);
6325 }
6326
6327 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6328                                     int ring_grps, int cp_rings, int stats,
6329                                     int vnics)
6330 {
6331         struct hwrm_func_cfg_input req = {0};
6332         u32 flags;
6333
6334         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6335                                      cp_rings, stats, vnics);
6336         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6337         if (BNXT_NEW_RM(bp)) {
6338                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6339                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6340                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6341                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6342                 if (bp->flags & BNXT_FLAG_CHIP_P5)
6343                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6344                                  FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6345                 else
6346                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6347         }
6348
6349         req.flags = cpu_to_le32(flags);
6350         return hwrm_send_message_silent(bp, &req, sizeof(req),
6351                                         HWRM_CMD_TIMEOUT);
6352 }
6353
6354 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6355                                  int ring_grps, int cp_rings, int stats,
6356                                  int vnics)
6357 {
6358         if (bp->hwrm_spec_code < 0x10801)
6359                 return 0;
6360
6361         if (BNXT_PF(bp))
6362                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6363                                                 ring_grps, cp_rings, stats,
6364                                                 vnics);
6365
6366         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6367                                         cp_rings, stats, vnics);
6368 }
6369
6370 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6371 {
6372         struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6373         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6374         struct hwrm_ring_aggint_qcaps_input req = {0};
6375         int rc;
6376
6377         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6378         coal_cap->num_cmpl_dma_aggr_max = 63;
6379         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6380         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6381         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6382         coal_cap->int_lat_tmr_min_max = 65535;
6383         coal_cap->int_lat_tmr_max_max = 65535;
6384         coal_cap->num_cmpl_aggr_int_max = 65535;
6385         coal_cap->timer_units = 80;
6386
6387         if (bp->hwrm_spec_code < 0x10902)
6388                 return;
6389
6390         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
6391         mutex_lock(&bp->hwrm_cmd_lock);
6392         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6393         if (!rc) {
6394                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6395                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6396                 coal_cap->num_cmpl_dma_aggr_max =
6397                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6398                 coal_cap->num_cmpl_dma_aggr_during_int_max =
6399                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6400                 coal_cap->cmpl_aggr_dma_tmr_max =
6401                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6402                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6403                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6404                 coal_cap->int_lat_tmr_min_max =
6405                         le16_to_cpu(resp->int_lat_tmr_min_max);
6406                 coal_cap->int_lat_tmr_max_max =
6407                         le16_to_cpu(resp->int_lat_tmr_max_max);
6408                 coal_cap->num_cmpl_aggr_int_max =
6409                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
6410                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6411         }
6412         mutex_unlock(&bp->hwrm_cmd_lock);
6413 }
6414
6415 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6416 {
6417         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6418
6419         return usec * 1000 / coal_cap->timer_units;
6420 }
6421
6422 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6423         struct bnxt_coal *hw_coal,
6424         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6425 {
6426         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6427         u32 cmpl_params = coal_cap->cmpl_params;
6428         u16 val, tmr, max, flags = 0;
6429
6430         max = hw_coal->bufs_per_record * 128;
6431         if (hw_coal->budget)
6432                 max = hw_coal->bufs_per_record * hw_coal->budget;
6433         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6434
6435         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6436         req->num_cmpl_aggr_int = cpu_to_le16(val);
6437
6438         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6439         req->num_cmpl_dma_aggr = cpu_to_le16(val);
6440
6441         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6442                       coal_cap->num_cmpl_dma_aggr_during_int_max);
6443         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6444
6445         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6446         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6447         req->int_lat_tmr_max = cpu_to_le16(tmr);
6448
6449         /* min timer set to 1/2 of interrupt timer */
6450         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6451                 val = tmr / 2;
6452                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6453                 req->int_lat_tmr_min = cpu_to_le16(val);
6454                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6455         }
6456
6457         /* buf timer set to 1/4 of interrupt timer */
6458         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6459         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6460
6461         if (cmpl_params &
6462             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6463                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6464                 val = clamp_t(u16, tmr, 1,
6465                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6466                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6467                 req->enables |=
6468                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6469         }
6470
6471         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
6472                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
6473         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6474             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6475                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6476         req->flags = cpu_to_le16(flags);
6477         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6478 }
6479
6480 /* Caller holds bp->hwrm_cmd_lock */
6481 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6482                                    struct bnxt_coal *hw_coal)
6483 {
6484         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
6485         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6486         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6487         u32 nq_params = coal_cap->nq_params;
6488         u16 tmr;
6489
6490         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6491                 return 0;
6492
6493         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
6494                                -1, -1);
6495         req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6496         req.flags =
6497                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6498
6499         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6500         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6501         req.int_lat_tmr_min = cpu_to_le16(tmr);
6502         req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6503         return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6504 }
6505
6506 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6507 {
6508         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
6509         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6510         struct bnxt_coal coal;
6511
6512         /* Tick values in micro seconds.
6513          * 1 coal_buf x bufs_per_record = 1 completion record.
6514          */
6515         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6516
6517         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6518         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6519
6520         if (!bnapi->rx_ring)
6521                 return -ENODEV;
6522
6523         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6524                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6525
6526         bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6527
6528         req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6529
6530         return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6531                                  HWRM_CMD_TIMEOUT);
6532 }
6533
6534 int bnxt_hwrm_set_coal(struct bnxt *bp)
6535 {
6536         int i, rc = 0;
6537         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6538                                                            req_tx = {0}, *req;
6539
6540         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6541                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6542         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6543                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6544
6545         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6546         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
6547
6548         mutex_lock(&bp->hwrm_cmd_lock);
6549         for (i = 0; i < bp->cp_nr_rings; i++) {
6550                 struct bnxt_napi *bnapi = bp->bnapi[i];
6551                 struct bnxt_coal *hw_coal;
6552                 u16 ring_id;
6553
6554                 req = &req_rx;
6555                 if (!bnapi->rx_ring) {
6556                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6557                         req = &req_tx;
6558                 } else {
6559                         ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6560                 }
6561                 req->ring_id = cpu_to_le16(ring_id);
6562
6563                 rc = _hwrm_send_message(bp, req, sizeof(*req),
6564                                         HWRM_CMD_TIMEOUT);
6565                 if (rc)
6566                         break;
6567
6568                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6569                         continue;
6570
6571                 if (bnapi->rx_ring && bnapi->tx_ring) {
6572                         req = &req_tx;
6573                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6574                         req->ring_id = cpu_to_le16(ring_id);
6575                         rc = _hwrm_send_message(bp, req, sizeof(*req),
6576                                                 HWRM_CMD_TIMEOUT);
6577                         if (rc)
6578                                 break;
6579                 }
6580                 if (bnapi->rx_ring)
6581                         hw_coal = &bp->rx_coal;
6582                 else
6583                         hw_coal = &bp->tx_coal;
6584                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6585         }
6586         mutex_unlock(&bp->hwrm_cmd_lock);
6587         return rc;
6588 }
6589
6590 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6591 {
6592         struct hwrm_stat_ctx_clr_stats_input req0 = {0};
6593         struct hwrm_stat_ctx_free_input req = {0};
6594         int i;
6595
6596         if (!bp->bnapi)
6597                 return;
6598
6599         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6600                 return;
6601
6602         bnxt_hwrm_cmd_hdr_init(bp, &req0, HWRM_STAT_CTX_CLR_STATS, -1, -1);
6603         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6604
6605         mutex_lock(&bp->hwrm_cmd_lock);
6606         for (i = 0; i < bp->cp_nr_rings; i++) {
6607                 struct bnxt_napi *bnapi = bp->bnapi[i];
6608                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6609
6610                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6611                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6612                         if (BNXT_FW_MAJ(bp) <= 20) {
6613                                 req0.stat_ctx_id = req.stat_ctx_id;
6614                                 _hwrm_send_message(bp, &req0, sizeof(req0),
6615                                                    HWRM_CMD_TIMEOUT);
6616                         }
6617                         _hwrm_send_message(bp, &req, sizeof(req),
6618                                            HWRM_CMD_TIMEOUT);
6619
6620                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6621                 }
6622         }
6623         mutex_unlock(&bp->hwrm_cmd_lock);
6624 }
6625
6626 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6627 {
6628         int rc = 0, i;
6629         struct hwrm_stat_ctx_alloc_input req = {0};
6630         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6631
6632         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6633                 return 0;
6634
6635         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6636
6637         req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6638         req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6639
6640         mutex_lock(&bp->hwrm_cmd_lock);
6641         for (i = 0; i < bp->cp_nr_rings; i++) {
6642                 struct bnxt_napi *bnapi = bp->bnapi[i];
6643                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6644
6645                 req.stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6646
6647                 rc = _hwrm_send_message(bp, &req, sizeof(req),
6648                                         HWRM_CMD_TIMEOUT);
6649                 if (rc)
6650                         break;
6651
6652                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6653
6654                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6655         }
6656         mutex_unlock(&bp->hwrm_cmd_lock);
6657         return rc;
6658 }
6659
6660 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6661 {
6662         struct hwrm_func_qcfg_input req = {0};
6663         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6664         u32 min_db_offset = 0;
6665         u16 flags;
6666         int rc;
6667
6668         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6669         req.fid = cpu_to_le16(0xffff);
6670         mutex_lock(&bp->hwrm_cmd_lock);
6671         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6672         if (rc)
6673                 goto func_qcfg_exit;
6674
6675 #ifdef CONFIG_BNXT_SRIOV
6676         if (BNXT_VF(bp)) {
6677                 struct bnxt_vf_info *vf = &bp->vf;
6678
6679                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6680         } else {
6681                 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6682         }
6683 #endif
6684         flags = le16_to_cpu(resp->flags);
6685         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6686                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6687                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6688                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6689                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6690         }
6691         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6692                 bp->flags |= BNXT_FLAG_MULTI_HOST;
6693         if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
6694                 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
6695
6696         switch (resp->port_partition_type) {
6697         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6698         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6699         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6700                 bp->port_partition_type = resp->port_partition_type;
6701                 break;
6702         }
6703         if (bp->hwrm_spec_code < 0x10707 ||
6704             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6705                 bp->br_mode = BRIDGE_MODE_VEB;
6706         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6707                 bp->br_mode = BRIDGE_MODE_VEPA;
6708         else
6709                 bp->br_mode = BRIDGE_MODE_UNDEF;
6710
6711         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6712         if (!bp->max_mtu)
6713                 bp->max_mtu = BNXT_MAX_MTU;
6714
6715         if (bp->db_size)
6716                 goto func_qcfg_exit;
6717
6718         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6719                 if (BNXT_PF(bp))
6720                         min_db_offset = DB_PF_OFFSET_P5;
6721                 else
6722                         min_db_offset = DB_VF_OFFSET_P5;
6723         }
6724         bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
6725                                  1024);
6726         if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
6727             bp->db_size <= min_db_offset)
6728                 bp->db_size = pci_resource_len(bp->pdev, 2);
6729
6730 func_qcfg_exit:
6731         mutex_unlock(&bp->hwrm_cmd_lock);
6732         return rc;
6733 }
6734
6735 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6736 {
6737         struct hwrm_func_backing_store_qcaps_input req = {0};
6738         struct hwrm_func_backing_store_qcaps_output *resp =
6739                 bp->hwrm_cmd_resp_addr;
6740         int rc;
6741
6742         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6743                 return 0;
6744
6745         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6746         mutex_lock(&bp->hwrm_cmd_lock);
6747         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6748         if (!rc) {
6749                 struct bnxt_ctx_pg_info *ctx_pg;
6750                 struct bnxt_ctx_mem_info *ctx;
6751                 int i, tqm_rings;
6752
6753                 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6754                 if (!ctx) {
6755                         rc = -ENOMEM;
6756                         goto ctx_err;
6757                 }
6758                 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6759                 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6760                 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6761                 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6762                 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6763                 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6764                 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6765                 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6766                 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6767                 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6768                 ctx->vnic_max_vnic_entries =
6769                         le16_to_cpu(resp->vnic_max_vnic_entries);
6770                 ctx->vnic_max_ring_table_entries =
6771                         le16_to_cpu(resp->vnic_max_ring_table_entries);
6772                 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6773                 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6774                 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6775                 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6776                 ctx->tqm_min_entries_per_ring =
6777                         le32_to_cpu(resp->tqm_min_entries_per_ring);
6778                 ctx->tqm_max_entries_per_ring =
6779                         le32_to_cpu(resp->tqm_max_entries_per_ring);
6780                 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6781                 if (!ctx->tqm_entries_multiple)
6782                         ctx->tqm_entries_multiple = 1;
6783                 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6784                 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
6785                 ctx->mrav_num_entries_units =
6786                         le16_to_cpu(resp->mrav_num_entries_units);
6787                 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6788                 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6789                 ctx->ctx_kind_initializer = resp->ctx_kind_initializer;
6790                 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
6791                 if (!ctx->tqm_fp_rings_count)
6792                         ctx->tqm_fp_rings_count = bp->max_q;
6793
6794                 tqm_rings = ctx->tqm_fp_rings_count + 1;
6795                 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
6796                 if (!ctx_pg) {
6797                         kfree(ctx);
6798                         rc = -ENOMEM;
6799                         goto ctx_err;
6800                 }
6801                 for (i = 0; i < tqm_rings; i++, ctx_pg++)
6802                         ctx->tqm_mem[i] = ctx_pg;
6803                 bp->ctx = ctx;
6804         } else {
6805                 rc = 0;
6806         }
6807 ctx_err:
6808         mutex_unlock(&bp->hwrm_cmd_lock);
6809         return rc;
6810 }
6811
6812 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6813                                   __le64 *pg_dir)
6814 {
6815         u8 pg_size = 0;
6816
6817         if (BNXT_PAGE_SHIFT == 13)
6818                 pg_size = 1 << 4;
6819         else if (BNXT_PAGE_SIZE == 16)
6820                 pg_size = 2 << 4;
6821
6822         *pg_attr = pg_size;
6823         if (rmem->depth >= 1) {
6824                 if (rmem->depth == 2)
6825                         *pg_attr |= 2;
6826                 else
6827                         *pg_attr |= 1;
6828                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6829         } else {
6830                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6831         }
6832 }
6833
6834 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
6835         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
6836          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
6837          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
6838          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
6839          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6840
6841 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6842 {
6843         struct hwrm_func_backing_store_cfg_input req = {0};
6844         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6845         struct bnxt_ctx_pg_info *ctx_pg;
6846         __le32 *num_entries;
6847         __le64 *pg_dir;
6848         u32 flags = 0;
6849         u8 *pg_attr;
6850         u32 ena;
6851         int i;
6852
6853         if (!ctx)
6854                 return 0;
6855
6856         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6857         req.enables = cpu_to_le32(enables);
6858
6859         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6860                 ctx_pg = &ctx->qp_mem;
6861                 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6862                 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6863                 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6864                 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6865                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6866                                       &req.qpc_pg_size_qpc_lvl,
6867                                       &req.qpc_page_dir);
6868         }
6869         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6870                 ctx_pg = &ctx->srq_mem;
6871                 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6872                 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6873                 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6874                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6875                                       &req.srq_pg_size_srq_lvl,
6876                                       &req.srq_page_dir);
6877         }
6878         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6879                 ctx_pg = &ctx->cq_mem;
6880                 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6881                 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6882                 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6883                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6884                                       &req.cq_page_dir);
6885         }
6886         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6887                 ctx_pg = &ctx->vnic_mem;
6888                 req.vnic_num_vnic_entries =
6889                         cpu_to_le16(ctx->vnic_max_vnic_entries);
6890                 req.vnic_num_ring_table_entries =
6891                         cpu_to_le16(ctx->vnic_max_ring_table_entries);
6892                 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6893                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6894                                       &req.vnic_pg_size_vnic_lvl,
6895                                       &req.vnic_page_dir);
6896         }
6897         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6898                 ctx_pg = &ctx->stat_mem;
6899                 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6900                 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6901                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6902                                       &req.stat_pg_size_stat_lvl,
6903                                       &req.stat_page_dir);
6904         }
6905         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
6906                 ctx_pg = &ctx->mrav_mem;
6907                 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
6908                 if (ctx->mrav_num_entries_units)
6909                         flags |=
6910                         FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
6911                 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
6912                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6913                                       &req.mrav_pg_size_mrav_lvl,
6914                                       &req.mrav_page_dir);
6915         }
6916         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
6917                 ctx_pg = &ctx->tim_mem;
6918                 req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
6919                 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
6920                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6921                                       &req.tim_pg_size_tim_lvl,
6922                                       &req.tim_page_dir);
6923         }
6924         for (i = 0, num_entries = &req.tqm_sp_num_entries,
6925              pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6926              pg_dir = &req.tqm_sp_page_dir,
6927              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6928              i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6929                 if (!(enables & ena))
6930                         continue;
6931
6932                 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6933                 ctx_pg = ctx->tqm_mem[i];
6934                 *num_entries = cpu_to_le32(ctx_pg->entries);
6935                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6936         }
6937         req.flags = cpu_to_le32(flags);
6938         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6939 }
6940
6941 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
6942                                   struct bnxt_ctx_pg_info *ctx_pg)
6943 {
6944         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6945
6946         rmem->page_size = BNXT_PAGE_SIZE;
6947         rmem->pg_arr = ctx_pg->ctx_pg_arr;
6948         rmem->dma_arr = ctx_pg->ctx_dma_arr;
6949         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
6950         if (rmem->depth >= 1)
6951                 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
6952         return bnxt_alloc_ring(bp, rmem);
6953 }
6954
6955 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
6956                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
6957                                   u8 depth, bool use_init_val)
6958 {
6959         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6960         int rc;
6961
6962         if (!mem_size)
6963                 return -EINVAL;
6964
6965         ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6966         if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
6967                 ctx_pg->nr_pages = 0;
6968                 return -EINVAL;
6969         }
6970         if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
6971                 int nr_tbls, i;
6972
6973                 rmem->depth = 2;
6974                 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
6975                                              GFP_KERNEL);
6976                 if (!ctx_pg->ctx_pg_tbl)
6977                         return -ENOMEM;
6978                 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
6979                 rmem->nr_pages = nr_tbls;
6980                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6981                 if (rc)
6982                         return rc;
6983                 for (i = 0; i < nr_tbls; i++) {
6984                         struct bnxt_ctx_pg_info *pg_tbl;
6985
6986                         pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
6987                         if (!pg_tbl)
6988                                 return -ENOMEM;
6989                         ctx_pg->ctx_pg_tbl[i] = pg_tbl;
6990                         rmem = &pg_tbl->ring_mem;
6991                         rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
6992                         rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
6993                         rmem->depth = 1;
6994                         rmem->nr_pages = MAX_CTX_PAGES;
6995                         if (use_init_val)
6996                                 rmem->init_val = bp->ctx->ctx_kind_initializer;
6997                         if (i == (nr_tbls - 1)) {
6998                                 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
6999
7000                                 if (rem)
7001                                         rmem->nr_pages = rem;
7002                         }
7003                         rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
7004                         if (rc)
7005                                 break;
7006                 }
7007         } else {
7008                 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7009                 if (rmem->nr_pages > 1 || depth)
7010                         rmem->depth = 1;
7011                 if (use_init_val)
7012                         rmem->init_val = bp->ctx->ctx_kind_initializer;
7013                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7014         }
7015         return rc;
7016 }
7017
7018 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
7019                                   struct bnxt_ctx_pg_info *ctx_pg)
7020 {
7021         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7022
7023         if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
7024             ctx_pg->ctx_pg_tbl) {
7025                 int i, nr_tbls = rmem->nr_pages;
7026
7027                 for (i = 0; i < nr_tbls; i++) {
7028                         struct bnxt_ctx_pg_info *pg_tbl;
7029                         struct bnxt_ring_mem_info *rmem2;
7030
7031                         pg_tbl = ctx_pg->ctx_pg_tbl[i];
7032                         if (!pg_tbl)
7033                                 continue;
7034                         rmem2 = &pg_tbl->ring_mem;
7035                         bnxt_free_ring(bp, rmem2);
7036                         ctx_pg->ctx_pg_arr[i] = NULL;
7037                         kfree(pg_tbl);
7038                         ctx_pg->ctx_pg_tbl[i] = NULL;
7039                 }
7040                 kfree(ctx_pg->ctx_pg_tbl);
7041                 ctx_pg->ctx_pg_tbl = NULL;
7042         }
7043         bnxt_free_ring(bp, rmem);
7044         ctx_pg->nr_pages = 0;
7045 }
7046
7047 static void bnxt_free_ctx_mem(struct bnxt *bp)
7048 {
7049         struct bnxt_ctx_mem_info *ctx = bp->ctx;
7050         int i;
7051
7052         if (!ctx)
7053                 return;
7054
7055         if (ctx->tqm_mem[0]) {
7056                 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7057                         bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7058                 kfree(ctx->tqm_mem[0]);
7059                 ctx->tqm_mem[0] = NULL;
7060         }
7061
7062         bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7063         bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7064         bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7065         bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7066         bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7067         bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7068         bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7069         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7070 }
7071
7072 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7073 {
7074         struct bnxt_ctx_pg_info *ctx_pg;
7075         struct bnxt_ctx_mem_info *ctx;
7076         u32 mem_size, ena, entries;
7077         u32 entries_sp, min;
7078         u32 num_mr, num_ah;
7079         u32 extra_srqs = 0;
7080         u32 extra_qps = 0;
7081         u8 pg_lvl = 1;
7082         int i, rc;
7083
7084         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7085         if (rc) {
7086                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7087                            rc);
7088                 return rc;
7089         }
7090         ctx = bp->ctx;
7091         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7092                 return 0;
7093
7094         if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7095                 pg_lvl = 2;
7096                 extra_qps = 65536;
7097                 extra_srqs = 8192;
7098         }
7099
7100         ctx_pg = &ctx->qp_mem;
7101         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7102                           extra_qps;
7103         mem_size = ctx->qp_entry_size * ctx_pg->entries;
7104         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7105         if (rc)
7106                 return rc;
7107
7108         ctx_pg = &ctx->srq_mem;
7109         ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7110         mem_size = ctx->srq_entry_size * ctx_pg->entries;
7111         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7112         if (rc)
7113                 return rc;
7114
7115         ctx_pg = &ctx->cq_mem;
7116         ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7117         mem_size = ctx->cq_entry_size * ctx_pg->entries;
7118         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7119         if (rc)
7120                 return rc;
7121
7122         ctx_pg = &ctx->vnic_mem;
7123         ctx_pg->entries = ctx->vnic_max_vnic_entries +
7124                           ctx->vnic_max_ring_table_entries;
7125         mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7126         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7127         if (rc)
7128                 return rc;
7129
7130         ctx_pg = &ctx->stat_mem;
7131         ctx_pg->entries = ctx->stat_max_entries;
7132         mem_size = ctx->stat_entry_size * ctx_pg->entries;
7133         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7134         if (rc)
7135                 return rc;
7136
7137         ena = 0;
7138         if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7139                 goto skip_rdma;
7140
7141         ctx_pg = &ctx->mrav_mem;
7142         /* 128K extra is needed to accommodate static AH context
7143          * allocation by f/w.
7144          */
7145         num_mr = 1024 * 256;
7146         num_ah = 1024 * 128;
7147         ctx_pg->entries = num_mr + num_ah;
7148         mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7149         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, true);
7150         if (rc)
7151                 return rc;
7152         ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7153         if (ctx->mrav_num_entries_units)
7154                 ctx_pg->entries =
7155                         ((num_mr / ctx->mrav_num_entries_units) << 16) |
7156                          (num_ah / ctx->mrav_num_entries_units);
7157
7158         ctx_pg = &ctx->tim_mem;
7159         ctx_pg->entries = ctx->qp_mem.entries;
7160         mem_size = ctx->tim_entry_size * ctx_pg->entries;
7161         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7162         if (rc)
7163                 return rc;
7164         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7165
7166 skip_rdma:
7167         min = ctx->tqm_min_entries_per_ring;
7168         entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7169                      2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7170         entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7171         entries = ctx->qp_max_l2_entries + extra_qps + ctx->qp_min_qp1_entries;
7172         entries = roundup(entries, ctx->tqm_entries_multiple);
7173         entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7174         for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7175                 ctx_pg = ctx->tqm_mem[i];
7176                 ctx_pg->entries = i ? entries : entries_sp;
7177                 mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7178                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7179                 if (rc)
7180                         return rc;
7181                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7182         }
7183         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7184         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7185         if (rc) {
7186                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7187                            rc);
7188                 return rc;
7189         }
7190         ctx->flags |= BNXT_CTX_FLAG_INITED;
7191         return 0;
7192 }
7193
7194 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7195 {
7196         struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7197         struct hwrm_func_resource_qcaps_input req = {0};
7198         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7199         int rc;
7200
7201         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
7202         req.fid = cpu_to_le16(0xffff);
7203
7204         mutex_lock(&bp->hwrm_cmd_lock);
7205         rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
7206                                        HWRM_CMD_TIMEOUT);
7207         if (rc)
7208                 goto hwrm_func_resc_qcaps_exit;
7209
7210         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7211         if (!all)
7212                 goto hwrm_func_resc_qcaps_exit;
7213
7214         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7215         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7216         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7217         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7218         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7219         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7220         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7221         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7222         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7223         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7224         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7225         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7226         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7227         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7228         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7229         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7230
7231         if (bp->flags & BNXT_FLAG_CHIP_P5) {
7232                 u16 max_msix = le16_to_cpu(resp->max_msix);
7233
7234                 hw_resc->max_nqs = max_msix;
7235                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7236         }
7237
7238         if (BNXT_PF(bp)) {
7239                 struct bnxt_pf_info *pf = &bp->pf;
7240
7241                 pf->vf_resv_strategy =
7242                         le16_to_cpu(resp->vf_reservation_strategy);
7243                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7244                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7245         }
7246 hwrm_func_resc_qcaps_exit:
7247         mutex_unlock(&bp->hwrm_cmd_lock);
7248         return rc;
7249 }
7250
7251 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7252 {
7253         int rc = 0;
7254         struct hwrm_func_qcaps_input req = {0};
7255         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7256         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7257         u32 flags, flags_ext;
7258
7259         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
7260         req.fid = cpu_to_le16(0xffff);
7261
7262         mutex_lock(&bp->hwrm_cmd_lock);
7263         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7264         if (rc)
7265                 goto hwrm_func_qcaps_exit;
7266
7267         flags = le32_to_cpu(resp->flags);
7268         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7269                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7270         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7271                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7272         if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7273                 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7274         if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7275                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7276         if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7277                 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7278         if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7279                 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7280         if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7281                 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7282         if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7283                 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7284
7285         flags_ext = le32_to_cpu(resp->flags_ext);
7286         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7287                 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7288
7289         bp->tx_push_thresh = 0;
7290         if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7291             BNXT_FW_MAJ(bp) > 217)
7292                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7293
7294         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7295         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7296         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7297         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7298         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7299         if (!hw_resc->max_hw_ring_grps)
7300                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7301         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7302         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7303         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7304
7305         if (BNXT_PF(bp)) {
7306                 struct bnxt_pf_info *pf = &bp->pf;
7307
7308                 pf->fw_fid = le16_to_cpu(resp->fid);
7309                 pf->port_id = le16_to_cpu(resp->port_id);
7310                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7311                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7312                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
7313                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7314                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7315                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7316                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7317                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7318                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7319                 bp->flags &= ~BNXT_FLAG_WOL_CAP;
7320                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7321                         bp->flags |= BNXT_FLAG_WOL_CAP;
7322         } else {
7323 #ifdef CONFIG_BNXT_SRIOV
7324                 struct bnxt_vf_info *vf = &bp->vf;
7325
7326                 vf->fw_fid = le16_to_cpu(resp->fid);
7327                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7328 #endif
7329         }
7330
7331 hwrm_func_qcaps_exit:
7332         mutex_unlock(&bp->hwrm_cmd_lock);
7333         return rc;
7334 }
7335
7336 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7337
7338 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7339 {
7340         int rc;
7341
7342         rc = __bnxt_hwrm_func_qcaps(bp);
7343         if (rc)
7344                 return rc;
7345         rc = bnxt_hwrm_queue_qportcfg(bp);
7346         if (rc) {
7347                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7348                 return rc;
7349         }
7350         if (bp->hwrm_spec_code >= 0x10803) {
7351                 rc = bnxt_alloc_ctx_mem(bp);
7352                 if (rc)
7353                         return rc;
7354                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7355                 if (!rc)
7356                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7357         }
7358         return 0;
7359 }
7360
7361 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7362 {
7363         struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
7364         struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7365         int rc = 0;
7366         u32 flags;
7367
7368         if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7369                 return 0;
7370
7371         resp = bp->hwrm_cmd_resp_addr;
7372         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
7373
7374         mutex_lock(&bp->hwrm_cmd_lock);
7375         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7376         if (rc)
7377                 goto hwrm_cfa_adv_qcaps_exit;
7378
7379         flags = le32_to_cpu(resp->flags);
7380         if (flags &
7381             CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7382                 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7383
7384 hwrm_cfa_adv_qcaps_exit:
7385         mutex_unlock(&bp->hwrm_cmd_lock);
7386         return rc;
7387 }
7388
7389 static int __bnxt_alloc_fw_health(struct bnxt *bp)
7390 {
7391         if (bp->fw_health)
7392                 return 0;
7393
7394         bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
7395         if (!bp->fw_health)
7396                 return -ENOMEM;
7397
7398         return 0;
7399 }
7400
7401 static int bnxt_alloc_fw_health(struct bnxt *bp)
7402 {
7403         int rc;
7404
7405         if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
7406             !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7407                 return 0;
7408
7409         rc = __bnxt_alloc_fw_health(bp);
7410         if (rc) {
7411                 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
7412                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7413                 return rc;
7414         }
7415
7416         return 0;
7417 }
7418
7419 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
7420 {
7421         writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
7422                                          BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7423                                          BNXT_FW_HEALTH_WIN_MAP_OFF);
7424 }
7425
7426 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
7427 {
7428         void __iomem *hs;
7429         u32 status_loc;
7430         u32 reg_type;
7431         u32 sig;
7432
7433         __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
7434         hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
7435
7436         sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
7437         if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
7438                 if (bp->fw_health)
7439                         bp->fw_health->status_reliable = false;
7440                 return;
7441         }
7442
7443         if (__bnxt_alloc_fw_health(bp)) {
7444                 netdev_warn(bp->dev, "no memory for firmware status checks\n");
7445                 return;
7446         }
7447
7448         status_loc = readl(hs + offsetof(struct hcomm_status, fw_status_loc));
7449         bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
7450         reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
7451         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
7452                 __bnxt_map_fw_health_reg(bp, status_loc);
7453                 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
7454                         BNXT_FW_HEALTH_WIN_OFF(status_loc);
7455         }
7456
7457         bp->fw_health->status_reliable = true;
7458 }
7459
7460 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7461 {
7462         struct bnxt_fw_health *fw_health = bp->fw_health;
7463         u32 reg_base = 0xffffffff;
7464         int i;
7465
7466         /* Only pre-map the monitoring GRC registers using window 3 */
7467         for (i = 0; i < 4; i++) {
7468                 u32 reg = fw_health->regs[i];
7469
7470                 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7471                         continue;
7472                 if (reg_base == 0xffffffff)
7473                         reg_base = reg & BNXT_GRC_BASE_MASK;
7474                 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7475                         return -ERANGE;
7476                 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
7477         }
7478         if (reg_base == 0xffffffff)
7479                 return 0;
7480
7481         __bnxt_map_fw_health_reg(bp, reg_base);
7482         return 0;
7483 }
7484
7485 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7486 {
7487         struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7488         struct bnxt_fw_health *fw_health = bp->fw_health;
7489         struct hwrm_error_recovery_qcfg_input req = {0};
7490         int rc, i;
7491
7492         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7493                 return 0;
7494
7495         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1);
7496         mutex_lock(&bp->hwrm_cmd_lock);
7497         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7498         if (rc)
7499                 goto err_recovery_out;
7500         fw_health->flags = le32_to_cpu(resp->flags);
7501         if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
7502             !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
7503                 rc = -EINVAL;
7504                 goto err_recovery_out;
7505         }
7506         fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
7507         fw_health->master_func_wait_dsecs =
7508                 le32_to_cpu(resp->master_func_wait_period);
7509         fw_health->normal_func_wait_dsecs =
7510                 le32_to_cpu(resp->normal_func_wait_period);
7511         fw_health->post_reset_wait_dsecs =
7512                 le32_to_cpu(resp->master_func_wait_period_after_reset);
7513         fw_health->post_reset_max_wait_dsecs =
7514                 le32_to_cpu(resp->max_bailout_time_after_reset);
7515         fw_health->regs[BNXT_FW_HEALTH_REG] =
7516                 le32_to_cpu(resp->fw_health_status_reg);
7517         fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
7518                 le32_to_cpu(resp->fw_heartbeat_reg);
7519         fw_health->regs[BNXT_FW_RESET_CNT_REG] =
7520                 le32_to_cpu(resp->fw_reset_cnt_reg);
7521         fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
7522                 le32_to_cpu(resp->reset_inprogress_reg);
7523         fw_health->fw_reset_inprog_reg_mask =
7524                 le32_to_cpu(resp->reset_inprogress_reg_mask);
7525         fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
7526         if (fw_health->fw_reset_seq_cnt >= 16) {
7527                 rc = -EINVAL;
7528                 goto err_recovery_out;
7529         }
7530         for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
7531                 fw_health->fw_reset_seq_regs[i] =
7532                         le32_to_cpu(resp->reset_reg[i]);
7533                 fw_health->fw_reset_seq_vals[i] =
7534                         le32_to_cpu(resp->reset_reg_val[i]);
7535                 fw_health->fw_reset_seq_delay_msec[i] =
7536                         resp->delay_after_reset[i];
7537         }
7538 err_recovery_out:
7539         mutex_unlock(&bp->hwrm_cmd_lock);
7540         if (!rc)
7541                 rc = bnxt_map_fw_health_regs(bp);
7542         if (rc)
7543                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7544         return rc;
7545 }
7546
7547 static int bnxt_hwrm_func_reset(struct bnxt *bp)
7548 {
7549         struct hwrm_func_reset_input req = {0};
7550
7551         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
7552         req.enables = 0;
7553
7554         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
7555 }
7556
7557 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
7558 {
7559         struct hwrm_nvm_get_dev_info_output nvm_info;
7560
7561         if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
7562                 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
7563                          nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
7564                          nvm_info.nvm_cfg_ver_upd);
7565 }
7566
7567 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
7568 {
7569         int rc = 0;
7570         struct hwrm_queue_qportcfg_input req = {0};
7571         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
7572         u8 i, j, *qptr;
7573         bool no_rdma;
7574
7575         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
7576
7577         mutex_lock(&bp->hwrm_cmd_lock);
7578         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7579         if (rc)
7580                 goto qportcfg_exit;
7581
7582         if (!resp->max_configurable_queues) {
7583                 rc = -EINVAL;
7584                 goto qportcfg_exit;
7585         }
7586         bp->max_tc = resp->max_configurable_queues;
7587         bp->max_lltc = resp->max_configurable_lossless_queues;
7588         if (bp->max_tc > BNXT_MAX_QUEUE)
7589                 bp->max_tc = BNXT_MAX_QUEUE;
7590
7591         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
7592         qptr = &resp->queue_id0;
7593         for (i = 0, j = 0; i < bp->max_tc; i++) {
7594                 bp->q_info[j].queue_id = *qptr;
7595                 bp->q_ids[i] = *qptr++;
7596                 bp->q_info[j].queue_profile = *qptr++;
7597                 bp->tc_to_qidx[j] = j;
7598                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
7599                     (no_rdma && BNXT_PF(bp)))
7600                         j++;
7601         }
7602         bp->max_q = bp->max_tc;
7603         bp->max_tc = max_t(u8, j, 1);
7604
7605         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
7606                 bp->max_tc = 1;
7607
7608         if (bp->max_lltc > bp->max_tc)
7609                 bp->max_lltc = bp->max_tc;
7610
7611 qportcfg_exit:
7612         mutex_unlock(&bp->hwrm_cmd_lock);
7613         return rc;
7614 }
7615
7616 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent)
7617 {
7618         struct hwrm_ver_get_input req = {0};
7619         int rc;
7620
7621         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
7622         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
7623         req.hwrm_intf_min = HWRM_VERSION_MINOR;
7624         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
7625
7626         rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT,
7627                                    silent);
7628         return rc;
7629 }
7630
7631 static int bnxt_hwrm_ver_get(struct bnxt *bp)
7632 {
7633         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
7634         u16 fw_maj, fw_min, fw_bld, fw_rsv;
7635         u32 dev_caps_cfg, hwrm_ver;
7636         int rc, len;
7637
7638         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
7639         mutex_lock(&bp->hwrm_cmd_lock);
7640         rc = __bnxt_hwrm_ver_get(bp, false);
7641         if (rc)
7642                 goto hwrm_ver_get_exit;
7643
7644         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
7645
7646         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
7647                              resp->hwrm_intf_min_8b << 8 |
7648                              resp->hwrm_intf_upd_8b;
7649         if (resp->hwrm_intf_maj_8b < 1) {
7650                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
7651                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7652                             resp->hwrm_intf_upd_8b);
7653                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
7654         }
7655
7656         hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
7657                         HWRM_VERSION_UPDATE;
7658
7659         if (bp->hwrm_spec_code > hwrm_ver)
7660                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7661                          HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
7662                          HWRM_VERSION_UPDATE);
7663         else
7664                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7665                          resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7666                          resp->hwrm_intf_upd_8b);
7667
7668         fw_maj = le16_to_cpu(resp->hwrm_fw_major);
7669         if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
7670                 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
7671                 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
7672                 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
7673                 len = FW_VER_STR_LEN;
7674         } else {
7675                 fw_maj = resp->hwrm_fw_maj_8b;
7676                 fw_min = resp->hwrm_fw_min_8b;
7677                 fw_bld = resp->hwrm_fw_bld_8b;
7678                 fw_rsv = resp->hwrm_fw_rsvd_8b;
7679                 len = BC_HWRM_STR_LEN;
7680         }
7681         bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
7682         snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
7683                  fw_rsv);
7684
7685         if (strlen(resp->active_pkg_name)) {
7686                 int fw_ver_len = strlen(bp->fw_ver_str);
7687
7688                 snprintf(bp->fw_ver_str + fw_ver_len,
7689                          FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
7690                          resp->active_pkg_name);
7691                 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
7692         }
7693
7694         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
7695         if (!bp->hwrm_cmd_timeout)
7696                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
7697
7698         if (resp->hwrm_intf_maj_8b >= 1) {
7699                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
7700                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
7701         }
7702         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
7703                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
7704
7705         bp->chip_num = le16_to_cpu(resp->chip_num);
7706         bp->chip_rev = resp->chip_rev;
7707         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
7708             !resp->chip_metal)
7709                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
7710
7711         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
7712         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
7713             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
7714                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
7715
7716         if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
7717                 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
7718
7719         if (dev_caps_cfg &
7720             VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
7721                 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
7722
7723         if (dev_caps_cfg &
7724             VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
7725                 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
7726
7727         if (dev_caps_cfg &
7728             VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
7729                 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
7730
7731 hwrm_ver_get_exit:
7732         mutex_unlock(&bp->hwrm_cmd_lock);
7733         return rc;
7734 }
7735
7736 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
7737 {
7738         struct hwrm_fw_set_time_input req = {0};
7739         struct tm tm;
7740         time64_t now = ktime_get_real_seconds();
7741
7742         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
7743             bp->hwrm_spec_code < 0x10400)
7744                 return -EOPNOTSUPP;
7745
7746         time64_to_tm(now, 0, &tm);
7747         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
7748         req.year = cpu_to_le16(1900 + tm.tm_year);
7749         req.month = 1 + tm.tm_mon;
7750         req.day = tm.tm_mday;
7751         req.hour = tm.tm_hour;
7752         req.minute = tm.tm_min;
7753         req.second = tm.tm_sec;
7754         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7755 }
7756
7757 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
7758 {
7759         u64 sw_tmp;
7760
7761         hw &= mask;
7762         sw_tmp = (*sw & ~mask) | hw;
7763         if (hw < (*sw & mask))
7764                 sw_tmp += mask + 1;
7765         WRITE_ONCE(*sw, sw_tmp);
7766 }
7767
7768 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
7769                                     int count, bool ignore_zero)
7770 {
7771         int i;
7772
7773         for (i = 0; i < count; i++) {
7774                 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
7775
7776                 if (ignore_zero && !hw)
7777                         continue;
7778
7779                 if (masks[i] == -1ULL)
7780                         sw_stats[i] = hw;
7781                 else
7782                         bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
7783         }
7784 }
7785
7786 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
7787 {
7788         if (!stats->hw_stats)
7789                 return;
7790
7791         __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7792                                 stats->hw_masks, stats->len / 8, false);
7793 }
7794
7795 static void bnxt_accumulate_all_stats(struct bnxt *bp)
7796 {
7797         struct bnxt_stats_mem *ring0_stats;
7798         bool ignore_zero = false;
7799         int i;
7800
7801         /* Chip bug.  Counter intermittently becomes 0. */
7802         if (bp->flags & BNXT_FLAG_CHIP_P5)
7803                 ignore_zero = true;
7804
7805         for (i = 0; i < bp->cp_nr_rings; i++) {
7806                 struct bnxt_napi *bnapi = bp->bnapi[i];
7807                 struct bnxt_cp_ring_info *cpr;
7808                 struct bnxt_stats_mem *stats;
7809
7810                 cpr = &bnapi->cp_ring;
7811                 stats = &cpr->stats;
7812                 if (!i)
7813                         ring0_stats = stats;
7814                 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7815                                         ring0_stats->hw_masks,
7816                                         ring0_stats->len / 8, ignore_zero);
7817         }
7818         if (bp->flags & BNXT_FLAG_PORT_STATS) {
7819                 struct bnxt_stats_mem *stats = &bp->port_stats;
7820                 __le64 *hw_stats = stats->hw_stats;
7821                 u64 *sw_stats = stats->sw_stats;
7822                 u64 *masks = stats->hw_masks;
7823                 int cnt;
7824
7825                 cnt = sizeof(struct rx_port_stats) / 8;
7826                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7827
7828                 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7829                 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7830                 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7831                 cnt = sizeof(struct tx_port_stats) / 8;
7832                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7833         }
7834         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
7835                 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
7836                 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
7837         }
7838 }
7839
7840 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
7841 {
7842         struct bnxt_pf_info *pf = &bp->pf;
7843         struct hwrm_port_qstats_input req = {0};
7844
7845         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
7846                 return 0;
7847
7848         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7849                 return -EOPNOTSUPP;
7850
7851         req.flags = flags;
7852         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
7853         req.port_id = cpu_to_le16(pf->port_id);
7854         req.tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
7855                                             BNXT_TX_PORT_STATS_BYTE_OFFSET);
7856         req.rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
7857         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7858 }
7859
7860 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
7861 {
7862         struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
7863         struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
7864         struct hwrm_port_qstats_ext_input req = {0};
7865         struct bnxt_pf_info *pf = &bp->pf;
7866         u32 tx_stat_size;
7867         int rc;
7868
7869         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
7870                 return 0;
7871
7872         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7873                 return -EOPNOTSUPP;
7874
7875         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
7876         req.flags = flags;
7877         req.port_id = cpu_to_le16(pf->port_id);
7878         req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
7879         req.rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
7880         tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
7881                        sizeof(struct tx_port_stats_ext) : 0;
7882         req.tx_stat_size = cpu_to_le16(tx_stat_size);
7883         req.tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
7884         mutex_lock(&bp->hwrm_cmd_lock);
7885         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7886         if (!rc) {
7887                 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
7888                 bp->fw_tx_stats_ext_size = tx_stat_size ?
7889                         le16_to_cpu(resp->tx_stat_size) / 8 : 0;
7890         } else {
7891                 bp->fw_rx_stats_ext_size = 0;
7892                 bp->fw_tx_stats_ext_size = 0;
7893         }
7894         if (flags)
7895                 goto qstats_done;
7896
7897         if (bp->fw_tx_stats_ext_size <=
7898             offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
7899                 mutex_unlock(&bp->hwrm_cmd_lock);
7900                 bp->pri2cos_valid = 0;
7901                 return rc;
7902         }
7903
7904         bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
7905         req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
7906
7907         rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
7908         if (!rc) {
7909                 struct hwrm_queue_pri2cos_qcfg_output *resp2;
7910                 u8 *pri2cos;
7911                 int i, j;
7912
7913                 resp2 = bp->hwrm_cmd_resp_addr;
7914                 pri2cos = &resp2->pri0_cos_queue_id;
7915                 for (i = 0; i < 8; i++) {
7916                         u8 queue_id = pri2cos[i];
7917                         u8 queue_idx;
7918
7919                         /* Per port queue IDs start from 0, 10, 20, etc */
7920                         queue_idx = queue_id % 10;
7921                         if (queue_idx > BNXT_MAX_QUEUE) {
7922                                 bp->pri2cos_valid = false;
7923                                 goto qstats_done;
7924                         }
7925                         for (j = 0; j < bp->max_q; j++) {
7926                                 if (bp->q_ids[j] == queue_id)
7927                                         bp->pri2cos_idx[i] = queue_idx;
7928                         }
7929                 }
7930                 bp->pri2cos_valid = 1;
7931         }
7932 qstats_done:
7933         mutex_unlock(&bp->hwrm_cmd_lock);
7934         return rc;
7935 }
7936
7937 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
7938 {
7939         if (bp->vxlan_fw_dst_port_id != INVALID_HW_RING_ID)
7940                 bnxt_hwrm_tunnel_dst_port_free(
7941                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7942         if (bp->nge_fw_dst_port_id != INVALID_HW_RING_ID)
7943                 bnxt_hwrm_tunnel_dst_port_free(
7944                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7945 }
7946
7947 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
7948 {
7949         int rc, i;
7950         u32 tpa_flags = 0;
7951
7952         if (set_tpa)
7953                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
7954         else if (BNXT_NO_FW_ACCESS(bp))
7955                 return 0;
7956         for (i = 0; i < bp->nr_vnics; i++) {
7957                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
7958                 if (rc) {
7959                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
7960                                    i, rc);
7961                         return rc;
7962                 }
7963         }
7964         return 0;
7965 }
7966
7967 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
7968 {
7969         int i;
7970
7971         for (i = 0; i < bp->nr_vnics; i++)
7972                 bnxt_hwrm_vnic_set_rss(bp, i, false);
7973 }
7974
7975 static void bnxt_clear_vnic(struct bnxt *bp)
7976 {
7977         if (!bp->vnic_info)
7978                 return;
7979
7980         bnxt_hwrm_clear_vnic_filter(bp);
7981         if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
7982                 /* clear all RSS setting before free vnic ctx */
7983                 bnxt_hwrm_clear_vnic_rss(bp);
7984                 bnxt_hwrm_vnic_ctx_free(bp);
7985         }
7986         /* before free the vnic, undo the vnic tpa settings */
7987         if (bp->flags & BNXT_FLAG_TPA)
7988                 bnxt_set_tpa(bp, false);
7989         bnxt_hwrm_vnic_free(bp);
7990         if (bp->flags & BNXT_FLAG_CHIP_P5)
7991                 bnxt_hwrm_vnic_ctx_free(bp);
7992 }
7993
7994 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
7995                                     bool irq_re_init)
7996 {
7997         bnxt_clear_vnic(bp);
7998         bnxt_hwrm_ring_free(bp, close_path);
7999         bnxt_hwrm_ring_grp_free(bp);
8000         if (irq_re_init) {
8001                 bnxt_hwrm_stat_ctx_free(bp);
8002                 bnxt_hwrm_free_tunnel_ports(bp);
8003         }
8004 }
8005
8006 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
8007 {
8008         struct hwrm_func_cfg_input req = {0};
8009
8010         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
8011         req.fid = cpu_to_le16(0xffff);
8012         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
8013         if (br_mode == BRIDGE_MODE_VEB)
8014                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
8015         else if (br_mode == BRIDGE_MODE_VEPA)
8016                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
8017         else
8018                 return -EINVAL;
8019         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8020 }
8021
8022 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
8023 {
8024         struct hwrm_func_cfg_input req = {0};
8025
8026         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
8027                 return 0;
8028
8029         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
8030         req.fid = cpu_to_le16(0xffff);
8031         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
8032         req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
8033         if (size == 128)
8034                 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
8035
8036         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8037 }
8038
8039 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8040 {
8041         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
8042         int rc;
8043
8044         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
8045                 goto skip_rss_ctx;
8046
8047         /* allocate context for vnic */
8048         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
8049         if (rc) {
8050                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8051                            vnic_id, rc);
8052                 goto vnic_setup_err;
8053         }
8054         bp->rsscos_nr_ctxs++;
8055
8056         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8057                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
8058                 if (rc) {
8059                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
8060                                    vnic_id, rc);
8061                         goto vnic_setup_err;
8062                 }
8063                 bp->rsscos_nr_ctxs++;
8064         }
8065
8066 skip_rss_ctx:
8067         /* configure default vnic, ring grp */
8068         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8069         if (rc) {
8070                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8071                            vnic_id, rc);
8072                 goto vnic_setup_err;
8073         }
8074
8075         /* Enable RSS hashing on vnic */
8076         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
8077         if (rc) {
8078                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
8079                            vnic_id, rc);
8080                 goto vnic_setup_err;
8081         }
8082
8083         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8084                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8085                 if (rc) {
8086                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8087                                    vnic_id, rc);
8088                 }
8089         }
8090
8091 vnic_setup_err:
8092         return rc;
8093 }
8094
8095 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
8096 {
8097         int rc, i, nr_ctxs;
8098
8099         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
8100         for (i = 0; i < nr_ctxs; i++) {
8101                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
8102                 if (rc) {
8103                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
8104                                    vnic_id, i, rc);
8105                         break;
8106                 }
8107                 bp->rsscos_nr_ctxs++;
8108         }
8109         if (i < nr_ctxs)
8110                 return -ENOMEM;
8111
8112         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
8113         if (rc) {
8114                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
8115                            vnic_id, rc);
8116                 return rc;
8117         }
8118         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8119         if (rc) {
8120                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8121                            vnic_id, rc);
8122                 return rc;
8123         }
8124         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8125                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8126                 if (rc) {
8127                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8128                                    vnic_id, rc);
8129                 }
8130         }
8131         return rc;
8132 }
8133
8134 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8135 {
8136         if (bp->flags & BNXT_FLAG_CHIP_P5)
8137                 return __bnxt_setup_vnic_p5(bp, vnic_id);
8138         else
8139                 return __bnxt_setup_vnic(bp, vnic_id);
8140 }
8141
8142 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8143 {
8144 #ifdef CONFIG_RFS_ACCEL
8145         int i, rc = 0;
8146
8147         if (bp->flags & BNXT_FLAG_CHIP_P5)
8148                 return 0;
8149
8150         for (i = 0; i < bp->rx_nr_rings; i++) {
8151                 struct bnxt_vnic_info *vnic;
8152                 u16 vnic_id = i + 1;
8153                 u16 ring_id = i;
8154
8155                 if (vnic_id >= bp->nr_vnics)
8156                         break;
8157
8158                 vnic = &bp->vnic_info[vnic_id];
8159                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
8160                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8161                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8162                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8163                 if (rc) {
8164                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8165                                    vnic_id, rc);
8166                         break;
8167                 }
8168                 rc = bnxt_setup_vnic(bp, vnic_id);
8169                 if (rc)
8170                         break;
8171         }
8172         return rc;
8173 #else
8174         return 0;
8175 #endif
8176 }
8177
8178 /* Allow PF and VF with default VLAN to be in promiscuous mode */
8179 static bool bnxt_promisc_ok(struct bnxt *bp)
8180 {
8181 #ifdef CONFIG_BNXT_SRIOV
8182         if (BNXT_VF(bp) && !bp->vf.vlan)
8183                 return false;
8184 #endif
8185         return true;
8186 }
8187
8188 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8189 {
8190         unsigned int rc = 0;
8191
8192         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8193         if (rc) {
8194                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8195                            rc);
8196                 return rc;
8197         }
8198
8199         rc = bnxt_hwrm_vnic_cfg(bp, 1);
8200         if (rc) {
8201                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8202                            rc);
8203                 return rc;
8204         }
8205         return rc;
8206 }
8207
8208 static int bnxt_cfg_rx_mode(struct bnxt *);
8209 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8210
8211 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8212 {
8213         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8214         int rc = 0;
8215         unsigned int rx_nr_rings = bp->rx_nr_rings;
8216
8217         if (irq_re_init) {
8218                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
8219                 if (rc) {
8220                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8221                                    rc);
8222                         goto err_out;
8223                 }
8224         }
8225
8226         rc = bnxt_hwrm_ring_alloc(bp);
8227         if (rc) {
8228                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8229                 goto err_out;
8230         }
8231
8232         rc = bnxt_hwrm_ring_grp_alloc(bp);
8233         if (rc) {
8234                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8235                 goto err_out;
8236         }
8237
8238         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8239                 rx_nr_rings--;
8240
8241         /* default vnic 0 */
8242         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8243         if (rc) {
8244                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8245                 goto err_out;
8246         }
8247
8248         rc = bnxt_setup_vnic(bp, 0);
8249         if (rc)
8250                 goto err_out;
8251
8252         if (bp->flags & BNXT_FLAG_RFS) {
8253                 rc = bnxt_alloc_rfs_vnics(bp);
8254                 if (rc)
8255                         goto err_out;
8256         }
8257
8258         if (bp->flags & BNXT_FLAG_TPA) {
8259                 rc = bnxt_set_tpa(bp, true);
8260                 if (rc)
8261                         goto err_out;
8262         }
8263
8264         if (BNXT_VF(bp))
8265                 bnxt_update_vf_mac(bp);
8266
8267         /* Filter for default vnic 0 */
8268         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8269         if (rc) {
8270                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8271                 goto err_out;
8272         }
8273         vnic->uc_filter_count = 1;
8274
8275         vnic->rx_mask = 0;
8276         if (bp->dev->flags & IFF_BROADCAST)
8277                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8278
8279         if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
8280                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8281
8282         if (bp->dev->flags & IFF_ALLMULTI) {
8283                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8284                 vnic->mc_list_count = 0;
8285         } else {
8286                 u32 mask = 0;
8287
8288                 bnxt_mc_list_updated(bp, &mask);
8289                 vnic->rx_mask |= mask;
8290         }
8291
8292         rc = bnxt_cfg_rx_mode(bp);
8293         if (rc)
8294                 goto err_out;
8295
8296         rc = bnxt_hwrm_set_coal(bp);
8297         if (rc)
8298                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8299                                 rc);
8300
8301         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8302                 rc = bnxt_setup_nitroa0_vnic(bp);
8303                 if (rc)
8304                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8305                                    rc);
8306         }
8307
8308         if (BNXT_VF(bp)) {
8309                 bnxt_hwrm_func_qcfg(bp);
8310                 netdev_update_features(bp->dev);
8311         }
8312
8313         return 0;
8314
8315 err_out:
8316         bnxt_hwrm_resource_free(bp, 0, true);
8317
8318         return rc;
8319 }
8320
8321 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8322 {
8323         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8324         return 0;
8325 }
8326
8327 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8328 {
8329         bnxt_init_cp_rings(bp);
8330         bnxt_init_rx_rings(bp);
8331         bnxt_init_tx_rings(bp);
8332         bnxt_init_ring_grps(bp, irq_re_init);
8333         bnxt_init_vnics(bp);
8334
8335         return bnxt_init_chip(bp, irq_re_init);
8336 }
8337
8338 static int bnxt_set_real_num_queues(struct bnxt *bp)
8339 {
8340         int rc;
8341         struct net_device *dev = bp->dev;
8342
8343         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8344                                           bp->tx_nr_rings_xdp);
8345         if (rc)
8346                 return rc;
8347
8348         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8349         if (rc)
8350                 return rc;
8351
8352 #ifdef CONFIG_RFS_ACCEL
8353         if (bp->flags & BNXT_FLAG_RFS)
8354                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8355 #endif
8356
8357         return rc;
8358 }
8359
8360 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8361                            bool shared)
8362 {
8363         int _rx = *rx, _tx = *tx;
8364
8365         if (shared) {
8366                 *rx = min_t(int, _rx, max);
8367                 *tx = min_t(int, _tx, max);
8368         } else {
8369                 if (max < 2)
8370                         return -ENOMEM;
8371
8372                 while (_rx + _tx > max) {
8373                         if (_rx > _tx && _rx > 1)
8374                                 _rx--;
8375                         else if (_tx > 1)
8376                                 _tx--;
8377                 }
8378                 *rx = _rx;
8379                 *tx = _tx;
8380         }
8381         return 0;
8382 }
8383
8384 static void bnxt_setup_msix(struct bnxt *bp)
8385 {
8386         const int len = sizeof(bp->irq_tbl[0].name);
8387         struct net_device *dev = bp->dev;
8388         int tcs, i;
8389
8390         tcs = netdev_get_num_tc(dev);
8391         if (tcs) {
8392                 int i, off, count;
8393
8394                 for (i = 0; i < tcs; i++) {
8395                         count = bp->tx_nr_rings_per_tc;
8396                         off = i * count;
8397                         netdev_set_tc_queue(dev, i, count, off);
8398                 }
8399         }
8400
8401         for (i = 0; i < bp->cp_nr_rings; i++) {
8402                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8403                 char *attr;
8404
8405                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8406                         attr = "TxRx";
8407                 else if (i < bp->rx_nr_rings)
8408                         attr = "rx";
8409                 else
8410                         attr = "tx";
8411
8412                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8413                          attr, i);
8414                 bp->irq_tbl[map_idx].handler = bnxt_msix;
8415         }
8416 }
8417
8418 static void bnxt_setup_inta(struct bnxt *bp)
8419 {
8420         const int len = sizeof(bp->irq_tbl[0].name);
8421
8422         if (netdev_get_num_tc(bp->dev))
8423                 netdev_reset_tc(bp->dev);
8424
8425         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
8426                  0);
8427         bp->irq_tbl[0].handler = bnxt_inta;
8428 }
8429
8430 static int bnxt_setup_int_mode(struct bnxt *bp)
8431 {
8432         int rc;
8433
8434         if (bp->flags & BNXT_FLAG_USING_MSIX)
8435                 bnxt_setup_msix(bp);
8436         else
8437                 bnxt_setup_inta(bp);
8438
8439         rc = bnxt_set_real_num_queues(bp);
8440         return rc;
8441 }
8442
8443 #ifdef CONFIG_RFS_ACCEL
8444 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
8445 {
8446         return bp->hw_resc.max_rsscos_ctxs;
8447 }
8448
8449 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
8450 {
8451         return bp->hw_resc.max_vnics;
8452 }
8453 #endif
8454
8455 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
8456 {
8457         return bp->hw_resc.max_stat_ctxs;
8458 }
8459
8460 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
8461 {
8462         return bp->hw_resc.max_cp_rings;
8463 }
8464
8465 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
8466 {
8467         unsigned int cp = bp->hw_resc.max_cp_rings;
8468
8469         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8470                 cp -= bnxt_get_ulp_msix_num(bp);
8471
8472         return cp;
8473 }
8474
8475 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
8476 {
8477         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8478
8479         if (bp->flags & BNXT_FLAG_CHIP_P5)
8480                 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
8481
8482         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
8483 }
8484
8485 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
8486 {
8487         bp->hw_resc.max_irqs = max_irqs;
8488 }
8489
8490 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
8491 {
8492         unsigned int cp;
8493
8494         cp = bnxt_get_max_func_cp_rings_for_en(bp);
8495         if (bp->flags & BNXT_FLAG_CHIP_P5)
8496                 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
8497         else
8498                 return cp - bp->cp_nr_rings;
8499 }
8500
8501 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
8502 {
8503         return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
8504 }
8505
8506 int bnxt_get_avail_msix(struct bnxt *bp, int num)
8507 {
8508         int max_cp = bnxt_get_max_func_cp_rings(bp);
8509         int max_irq = bnxt_get_max_func_irqs(bp);
8510         int total_req = bp->cp_nr_rings + num;
8511         int max_idx, avail_msix;
8512
8513         max_idx = bp->total_irqs;
8514         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8515                 max_idx = min_t(int, bp->total_irqs, max_cp);
8516         avail_msix = max_idx - bp->cp_nr_rings;
8517         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
8518                 return avail_msix;
8519
8520         if (max_irq < total_req) {
8521                 num = max_irq - bp->cp_nr_rings;
8522                 if (num <= 0)
8523                         return 0;
8524         }
8525         return num;
8526 }
8527
8528 static int bnxt_get_num_msix(struct bnxt *bp)
8529 {
8530         if (!BNXT_NEW_RM(bp))
8531                 return bnxt_get_max_func_irqs(bp);
8532
8533         return bnxt_nq_rings_in_use(bp);
8534 }
8535
8536 static int bnxt_init_msix(struct bnxt *bp)
8537 {
8538         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
8539         struct msix_entry *msix_ent;
8540
8541         total_vecs = bnxt_get_num_msix(bp);
8542         max = bnxt_get_max_func_irqs(bp);
8543         if (total_vecs > max)
8544                 total_vecs = max;
8545
8546         if (!total_vecs)
8547                 return 0;
8548
8549         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
8550         if (!msix_ent)
8551                 return -ENOMEM;
8552
8553         for (i = 0; i < total_vecs; i++) {
8554                 msix_ent[i].entry = i;
8555                 msix_ent[i].vector = 0;
8556         }
8557
8558         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
8559                 min = 2;
8560
8561         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
8562         ulp_msix = bnxt_get_ulp_msix_num(bp);
8563         if (total_vecs < 0 || total_vecs < ulp_msix) {
8564                 rc = -ENODEV;
8565                 goto msix_setup_exit;
8566         }
8567
8568         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
8569         if (bp->irq_tbl) {
8570                 for (i = 0; i < total_vecs; i++)
8571                         bp->irq_tbl[i].vector = msix_ent[i].vector;
8572
8573                 bp->total_irqs = total_vecs;
8574                 /* Trim rings based upon num of vectors allocated */
8575                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
8576                                      total_vecs - ulp_msix, min == 1);
8577                 if (rc)
8578                         goto msix_setup_exit;
8579
8580                 bp->cp_nr_rings = (min == 1) ?
8581                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
8582                                   bp->tx_nr_rings + bp->rx_nr_rings;
8583
8584         } else {
8585                 rc = -ENOMEM;
8586                 goto msix_setup_exit;
8587         }
8588         bp->flags |= BNXT_FLAG_USING_MSIX;
8589         kfree(msix_ent);
8590         return 0;
8591
8592 msix_setup_exit:
8593         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
8594         kfree(bp->irq_tbl);
8595         bp->irq_tbl = NULL;
8596         pci_disable_msix(bp->pdev);
8597         kfree(msix_ent);
8598         return rc;
8599 }
8600
8601 static int bnxt_init_inta(struct bnxt *bp)
8602 {
8603         bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
8604         if (!bp->irq_tbl)
8605                 return -ENOMEM;
8606
8607         bp->total_irqs = 1;
8608         bp->rx_nr_rings = 1;
8609         bp->tx_nr_rings = 1;
8610         bp->cp_nr_rings = 1;
8611         bp->flags |= BNXT_FLAG_SHARED_RINGS;
8612         bp->irq_tbl[0].vector = bp->pdev->irq;
8613         return 0;
8614 }
8615
8616 static int bnxt_init_int_mode(struct bnxt *bp)
8617 {
8618         int rc = 0;
8619
8620         if (bp->flags & BNXT_FLAG_MSIX_CAP)
8621                 rc = bnxt_init_msix(bp);
8622
8623         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
8624                 /* fallback to INTA */
8625                 rc = bnxt_init_inta(bp);
8626         }
8627         return rc;
8628 }
8629
8630 static void bnxt_clear_int_mode(struct bnxt *bp)
8631 {
8632         if (bp->flags & BNXT_FLAG_USING_MSIX)
8633                 pci_disable_msix(bp->pdev);
8634
8635         kfree(bp->irq_tbl);
8636         bp->irq_tbl = NULL;
8637         bp->flags &= ~BNXT_FLAG_USING_MSIX;
8638 }
8639
8640 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
8641 {
8642         int tcs = netdev_get_num_tc(bp->dev);
8643         bool irq_cleared = false;
8644         int rc;
8645
8646         if (!bnxt_need_reserve_rings(bp))
8647                 return 0;
8648
8649         if (irq_re_init && BNXT_NEW_RM(bp) &&
8650             bnxt_get_num_msix(bp) != bp->total_irqs) {
8651                 bnxt_ulp_irq_stop(bp);
8652                 bnxt_clear_int_mode(bp);
8653                 irq_cleared = true;
8654         }
8655         rc = __bnxt_reserve_rings(bp);
8656         if (irq_cleared) {
8657                 if (!rc)
8658                         rc = bnxt_init_int_mode(bp);
8659                 bnxt_ulp_irq_restart(bp, rc);
8660         }
8661         if (rc) {
8662                 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
8663                 return rc;
8664         }
8665         if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
8666                 netdev_err(bp->dev, "tx ring reservation failure\n");
8667                 netdev_reset_tc(bp->dev);
8668                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8669                 return -ENOMEM;
8670         }
8671         return 0;
8672 }
8673
8674 static void bnxt_free_irq(struct bnxt *bp)
8675 {
8676         struct bnxt_irq *irq;
8677         int i;
8678
8679 #ifdef CONFIG_RFS_ACCEL
8680         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
8681         bp->dev->rx_cpu_rmap = NULL;
8682 #endif
8683         if (!bp->irq_tbl || !bp->bnapi)
8684                 return;
8685
8686         for (i = 0; i < bp->cp_nr_rings; i++) {
8687                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8688
8689                 irq = &bp->irq_tbl[map_idx];
8690                 if (irq->requested) {
8691                         if (irq->have_cpumask) {
8692                                 irq_set_affinity_hint(irq->vector, NULL);
8693                                 free_cpumask_var(irq->cpu_mask);
8694                                 irq->have_cpumask = 0;
8695                         }
8696                         free_irq(irq->vector, bp->bnapi[i]);
8697                 }
8698
8699                 irq->requested = 0;
8700         }
8701 }
8702
8703 static int bnxt_request_irq(struct bnxt *bp)
8704 {
8705         int i, j, rc = 0;
8706         unsigned long flags = 0;
8707 #ifdef CONFIG_RFS_ACCEL
8708         struct cpu_rmap *rmap;
8709 #endif
8710
8711         rc = bnxt_setup_int_mode(bp);
8712         if (rc) {
8713                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
8714                            rc);
8715                 return rc;
8716         }
8717 #ifdef CONFIG_RFS_ACCEL
8718         rmap = bp->dev->rx_cpu_rmap;
8719 #endif
8720         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
8721                 flags = IRQF_SHARED;
8722
8723         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
8724                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8725                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
8726
8727 #ifdef CONFIG_RFS_ACCEL
8728                 if (rmap && bp->bnapi[i]->rx_ring) {
8729                         rc = irq_cpu_rmap_add(rmap, irq->vector);
8730                         if (rc)
8731                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
8732                                             j);
8733                         j++;
8734                 }
8735 #endif
8736                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
8737                                  bp->bnapi[i]);
8738                 if (rc)
8739                         break;
8740
8741                 irq->requested = 1;
8742
8743                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
8744                         int numa_node = dev_to_node(&bp->pdev->dev);
8745
8746                         irq->have_cpumask = 1;
8747                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
8748                                         irq->cpu_mask);
8749                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
8750                         if (rc) {
8751                                 netdev_warn(bp->dev,
8752                                             "Set affinity failed, IRQ = %d\n",
8753                                             irq->vector);
8754                                 break;
8755                         }
8756                 }
8757         }
8758         return rc;
8759 }
8760
8761 static void bnxt_del_napi(struct bnxt *bp)
8762 {
8763         int i;
8764
8765         if (!bp->bnapi)
8766                 return;
8767
8768         for (i = 0; i < bp->cp_nr_rings; i++) {
8769                 struct bnxt_napi *bnapi = bp->bnapi[i];
8770
8771                 __netif_napi_del(&bnapi->napi);
8772         }
8773         /* We called __netif_napi_del(), we need
8774          * to respect an RCU grace period before freeing napi structures.
8775          */
8776         synchronize_net();
8777 }
8778
8779 static void bnxt_init_napi(struct bnxt *bp)
8780 {
8781         int i;
8782         unsigned int cp_nr_rings = bp->cp_nr_rings;
8783         struct bnxt_napi *bnapi;
8784
8785         if (bp->flags & BNXT_FLAG_USING_MSIX) {
8786                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
8787
8788                 if (bp->flags & BNXT_FLAG_CHIP_P5)
8789                         poll_fn = bnxt_poll_p5;
8790                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8791                         cp_nr_rings--;
8792                 for (i = 0; i < cp_nr_rings; i++) {
8793                         bnapi = bp->bnapi[i];
8794                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
8795                 }
8796                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8797                         bnapi = bp->bnapi[cp_nr_rings];
8798                         netif_napi_add(bp->dev, &bnapi->napi,
8799                                        bnxt_poll_nitroa0, 64);
8800                 }
8801         } else {
8802                 bnapi = bp->bnapi[0];
8803                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
8804         }
8805 }
8806
8807 static void bnxt_disable_napi(struct bnxt *bp)
8808 {
8809         int i;
8810
8811         if (!bp->bnapi)
8812                 return;
8813
8814         for (i = 0; i < bp->cp_nr_rings; i++) {
8815                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8816
8817                 if (bp->bnapi[i]->rx_ring)
8818                         cancel_work_sync(&cpr->dim.work);
8819
8820                 napi_disable(&bp->bnapi[i]->napi);
8821         }
8822 }
8823
8824 static void bnxt_enable_napi(struct bnxt *bp)
8825 {
8826         int i;
8827
8828         for (i = 0; i < bp->cp_nr_rings; i++) {
8829                 struct bnxt_napi *bnapi = bp->bnapi[i];
8830                 struct bnxt_cp_ring_info *cpr;
8831
8832                 cpr = &bnapi->cp_ring;
8833                 if (bnapi->in_reset)
8834                         cpr->sw_stats.rx.rx_resets++;
8835                 bnapi->in_reset = false;
8836
8837                 if (bnapi->rx_ring) {
8838                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
8839                         cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
8840                 }
8841                 napi_enable(&bnapi->napi);
8842         }
8843 }
8844
8845 void bnxt_tx_disable(struct bnxt *bp)
8846 {
8847         int i;
8848         struct bnxt_tx_ring_info *txr;
8849
8850         if (bp->tx_ring) {
8851                 for (i = 0; i < bp->tx_nr_rings; i++) {
8852                         txr = &bp->tx_ring[i];
8853                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
8854                 }
8855         }
8856         /* Stop all TX queues */
8857         netif_tx_disable(bp->dev);
8858         netif_carrier_off(bp->dev);
8859 }
8860
8861 void bnxt_tx_enable(struct bnxt *bp)
8862 {
8863         int i;
8864         struct bnxt_tx_ring_info *txr;
8865
8866         for (i = 0; i < bp->tx_nr_rings; i++) {
8867                 txr = &bp->tx_ring[i];
8868                 txr->dev_state = 0;
8869         }
8870         netif_tx_wake_all_queues(bp->dev);
8871         if (bp->link_info.link_up)
8872                 netif_carrier_on(bp->dev);
8873 }
8874
8875 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
8876 {
8877         u8 active_fec = link_info->active_fec_sig_mode &
8878                         PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
8879
8880         switch (active_fec) {
8881         default:
8882         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
8883                 return "None";
8884         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
8885                 return "Clause 74 BaseR";
8886         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
8887                 return "Clause 91 RS(528,514)";
8888         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
8889                 return "Clause 91 RS544_1XN";
8890         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
8891                 return "Clause 91 RS(544,514)";
8892         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
8893                 return "Clause 91 RS272_1XN";
8894         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
8895                 return "Clause 91 RS(272,257)";
8896         }
8897 }
8898
8899 static void bnxt_report_link(struct bnxt *bp)
8900 {
8901         if (bp->link_info.link_up) {
8902                 const char *duplex;
8903                 const char *flow_ctrl;
8904                 u32 speed;
8905                 u16 fec;
8906
8907                 netif_carrier_on(bp->dev);
8908                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
8909                 if (speed == SPEED_UNKNOWN) {
8910                         netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
8911                         return;
8912                 }
8913                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
8914                         duplex = "full";
8915                 else
8916                         duplex = "half";
8917                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
8918                         flow_ctrl = "ON - receive & transmit";
8919                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
8920                         flow_ctrl = "ON - transmit";
8921                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
8922                         flow_ctrl = "ON - receive";
8923                 else
8924                         flow_ctrl = "none";
8925                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
8926                             speed, duplex, flow_ctrl);
8927                 if (bp->flags & BNXT_FLAG_EEE_CAP)
8928                         netdev_info(bp->dev, "EEE is %s\n",
8929                                     bp->eee.eee_active ? "active" :
8930                                                          "not active");
8931                 fec = bp->link_info.fec_cfg;
8932                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
8933                         netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
8934                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
8935                                     bnxt_report_fec(&bp->link_info));
8936         } else {
8937                 netif_carrier_off(bp->dev);
8938                 netdev_err(bp->dev, "NIC Link is Down\n");
8939         }
8940 }
8941
8942 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
8943 {
8944         if (!resp->supported_speeds_auto_mode &&
8945             !resp->supported_speeds_force_mode &&
8946             !resp->supported_pam4_speeds_auto_mode &&
8947             !resp->supported_pam4_speeds_force_mode)
8948                 return true;
8949         return false;
8950 }
8951
8952 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
8953 {
8954         int rc = 0;
8955         struct hwrm_port_phy_qcaps_input req = {0};
8956         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8957         struct bnxt_link_info *link_info = &bp->link_info;
8958
8959         bp->flags &= ~BNXT_FLAG_EEE_CAP;
8960         if (bp->test_info)
8961                 bp->test_info->flags &= ~(BNXT_TEST_FL_EXT_LPBK |
8962                                           BNXT_TEST_FL_AN_PHY_LPBK);
8963         if (bp->hwrm_spec_code < 0x10201)
8964                 return 0;
8965
8966         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
8967
8968         mutex_lock(&bp->hwrm_cmd_lock);
8969         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8970         if (rc)
8971                 goto hwrm_phy_qcaps_exit;
8972
8973         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
8974                 struct ethtool_eee *eee = &bp->eee;
8975                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
8976
8977                 bp->flags |= BNXT_FLAG_EEE_CAP;
8978                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8979                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
8980                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
8981                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
8982                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
8983         }
8984         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
8985                 if (bp->test_info)
8986                         bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
8987         }
8988         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED) {
8989                 if (bp->test_info)
8990                         bp->test_info->flags |= BNXT_TEST_FL_AN_PHY_LPBK;
8991         }
8992         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED) {
8993                 if (BNXT_PF(bp))
8994                         bp->fw_cap |= BNXT_FW_CAP_SHARED_PORT_CFG;
8995         }
8996         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_CUMULATIVE_COUNTERS_ON_RESET)
8997                 bp->fw_cap |= BNXT_FW_CAP_PORT_STATS_NO_RESET;
8998
8999         if (bp->hwrm_spec_code >= 0x10a01) {
9000                 if (bnxt_phy_qcaps_no_speed(resp)) {
9001                         link_info->phy_state = BNXT_PHY_STATE_DISABLED;
9002                         netdev_warn(bp->dev, "Ethernet link disabled\n");
9003                 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
9004                         link_info->phy_state = BNXT_PHY_STATE_ENABLED;
9005                         netdev_info(bp->dev, "Ethernet link enabled\n");
9006                         /* Phy re-enabled, reprobe the speeds */
9007                         link_info->support_auto_speeds = 0;
9008                         link_info->support_pam4_auto_speeds = 0;
9009                 }
9010         }
9011         if (resp->supported_speeds_auto_mode)
9012                 link_info->support_auto_speeds =
9013                         le16_to_cpu(resp->supported_speeds_auto_mode);
9014         if (resp->supported_pam4_speeds_auto_mode)
9015                 link_info->support_pam4_auto_speeds =
9016                         le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
9017
9018         bp->port_count = resp->port_cnt;
9019
9020 hwrm_phy_qcaps_exit:
9021         mutex_unlock(&bp->hwrm_cmd_lock);
9022         return rc;
9023 }
9024
9025 static bool bnxt_support_dropped(u16 advertising, u16 supported)
9026 {
9027         u16 diff = advertising ^ supported;
9028
9029         return ((supported | diff) != supported);
9030 }
9031
9032 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
9033 {
9034         int rc = 0;
9035         struct bnxt_link_info *link_info = &bp->link_info;
9036         struct hwrm_port_phy_qcfg_input req = {0};
9037         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9038         u8 link_up = link_info->link_up;
9039         bool support_changed = false;
9040
9041         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
9042
9043         mutex_lock(&bp->hwrm_cmd_lock);
9044         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9045         if (rc) {
9046                 mutex_unlock(&bp->hwrm_cmd_lock);
9047                 return rc;
9048         }
9049
9050         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
9051         link_info->phy_link_status = resp->link;
9052         link_info->duplex = resp->duplex_cfg;
9053         if (bp->hwrm_spec_code >= 0x10800)
9054                 link_info->duplex = resp->duplex_state;
9055         link_info->pause = resp->pause;
9056         link_info->auto_mode = resp->auto_mode;
9057         link_info->auto_pause_setting = resp->auto_pause;
9058         link_info->lp_pause = resp->link_partner_adv_pause;
9059         link_info->force_pause_setting = resp->force_pause;
9060         link_info->duplex_setting = resp->duplex_cfg;
9061         if (link_info->phy_link_status == BNXT_LINK_LINK)
9062                 link_info->link_speed = le16_to_cpu(resp->link_speed);
9063         else
9064                 link_info->link_speed = 0;
9065         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
9066         link_info->force_pam4_link_speed =
9067                 le16_to_cpu(resp->force_pam4_link_speed);
9068         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
9069         link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
9070         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
9071         link_info->auto_pam4_link_speeds =
9072                 le16_to_cpu(resp->auto_pam4_link_speed_mask);
9073         link_info->lp_auto_link_speeds =
9074                 le16_to_cpu(resp->link_partner_adv_speeds);
9075         link_info->lp_auto_pam4_link_speeds =
9076                 resp->link_partner_pam4_adv_speeds;
9077         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
9078         link_info->phy_ver[0] = resp->phy_maj;
9079         link_info->phy_ver[1] = resp->phy_min;
9080         link_info->phy_ver[2] = resp->phy_bld;
9081         link_info->media_type = resp->media_type;
9082         link_info->phy_type = resp->phy_type;
9083         link_info->transceiver = resp->xcvr_pkg_type;
9084         link_info->phy_addr = resp->eee_config_phy_addr &
9085                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
9086         link_info->module_status = resp->module_status;
9087
9088         if (bp->flags & BNXT_FLAG_EEE_CAP) {
9089                 struct ethtool_eee *eee = &bp->eee;
9090                 u16 fw_speeds;
9091
9092                 eee->eee_active = 0;
9093                 if (resp->eee_config_phy_addr &
9094                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
9095                         eee->eee_active = 1;
9096                         fw_speeds = le16_to_cpu(
9097                                 resp->link_partner_adv_eee_link_speed_mask);
9098                         eee->lp_advertised =
9099                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9100                 }
9101
9102                 /* Pull initial EEE config */
9103                 if (!chng_link_state) {
9104                         if (resp->eee_config_phy_addr &
9105                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
9106                                 eee->eee_enabled = 1;
9107
9108                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
9109                         eee->advertised =
9110                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9111
9112                         if (resp->eee_config_phy_addr &
9113                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
9114                                 __le32 tmr;
9115
9116                                 eee->tx_lpi_enabled = 1;
9117                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
9118                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
9119                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
9120                         }
9121                 }
9122         }
9123
9124         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
9125         if (bp->hwrm_spec_code >= 0x10504) {
9126                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
9127                 link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
9128         }
9129         /* TODO: need to add more logic to report VF link */
9130         if (chng_link_state) {
9131                 if (link_info->phy_link_status == BNXT_LINK_LINK)
9132                         link_info->link_up = 1;
9133                 else
9134                         link_info->link_up = 0;
9135                 if (link_up != link_info->link_up)
9136                         bnxt_report_link(bp);
9137         } else {
9138                 /* alwasy link down if not require to update link state */
9139                 link_info->link_up = 0;
9140         }
9141         mutex_unlock(&bp->hwrm_cmd_lock);
9142
9143         if (!BNXT_PHY_CFG_ABLE(bp))
9144                 return 0;
9145
9146         /* Check if any advertised speeds are no longer supported. The caller
9147          * holds the link_lock mutex, so we can modify link_info settings.
9148          */
9149         if (bnxt_support_dropped(link_info->advertising,
9150                                  link_info->support_auto_speeds)) {
9151                 link_info->advertising = link_info->support_auto_speeds;
9152                 support_changed = true;
9153         }
9154         if (bnxt_support_dropped(link_info->advertising_pam4,
9155                                  link_info->support_pam4_auto_speeds)) {
9156                 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
9157                 support_changed = true;
9158         }
9159         if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
9160                 bnxt_hwrm_set_link_setting(bp, true, false);
9161         return 0;
9162 }
9163
9164 static void bnxt_get_port_module_status(struct bnxt *bp)
9165 {
9166         struct bnxt_link_info *link_info = &bp->link_info;
9167         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
9168         u8 module_status;
9169
9170         if (bnxt_update_link(bp, true))
9171                 return;
9172
9173         module_status = link_info->module_status;
9174         switch (module_status) {
9175         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
9176         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
9177         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
9178                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
9179                             bp->pf.port_id);
9180                 if (bp->hwrm_spec_code >= 0x10201) {
9181                         netdev_warn(bp->dev, "Module part number %s\n",
9182                                     resp->phy_vendor_partnumber);
9183                 }
9184                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
9185                         netdev_warn(bp->dev, "TX is disabled\n");
9186                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
9187                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
9188         }
9189 }
9190
9191 static void
9192 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9193 {
9194         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
9195                 if (bp->hwrm_spec_code >= 0x10201)
9196                         req->auto_pause =
9197                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
9198                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9199                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
9200                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9201                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
9202                 req->enables |=
9203                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9204         } else {
9205                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9206                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
9207                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9208                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9209                 req->enables |=
9210                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9211                 if (bp->hwrm_spec_code >= 0x10201) {
9212                         req->auto_pause = req->force_pause;
9213                         req->enables |= cpu_to_le32(
9214                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9215                 }
9216         }
9217 }
9218
9219 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9220 {
9221         if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
9222                 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9223                 if (bp->link_info.advertising) {
9224                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9225                         req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
9226                 }
9227                 if (bp->link_info.advertising_pam4) {
9228                         req->enables |=
9229                                 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
9230                         req->auto_link_pam4_speed_mask =
9231                                 cpu_to_le16(bp->link_info.advertising_pam4);
9232                 }
9233                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9234                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9235         } else {
9236                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9237                 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
9238                         req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9239                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
9240                 } else {
9241                         req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9242                 }
9243         }
9244
9245         /* tell chimp that the setting takes effect immediately */
9246         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9247 }
9248
9249 int bnxt_hwrm_set_pause(struct bnxt *bp)
9250 {
9251         struct hwrm_port_phy_cfg_input req = {0};
9252         int rc;
9253
9254         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9255         bnxt_hwrm_set_pause_common(bp, &req);
9256
9257         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9258             bp->link_info.force_link_chng)
9259                 bnxt_hwrm_set_link_common(bp, &req);
9260
9261         mutex_lock(&bp->hwrm_cmd_lock);
9262         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9263         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9264                 /* since changing of pause setting doesn't trigger any link
9265                  * change event, the driver needs to update the current pause
9266                  * result upon successfully return of the phy_cfg command
9267                  */
9268                 bp->link_info.pause =
9269                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9270                 bp->link_info.auto_pause_setting = 0;
9271                 if (!bp->link_info.force_link_chng)
9272                         bnxt_report_link(bp);
9273         }
9274         bp->link_info.force_link_chng = false;
9275         mutex_unlock(&bp->hwrm_cmd_lock);
9276         return rc;
9277 }
9278
9279 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9280                               struct hwrm_port_phy_cfg_input *req)
9281 {
9282         struct ethtool_eee *eee = &bp->eee;
9283
9284         if (eee->eee_enabled) {
9285                 u16 eee_speeds;
9286                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9287
9288                 if (eee->tx_lpi_enabled)
9289                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9290                 else
9291                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9292
9293                 req->flags |= cpu_to_le32(flags);
9294                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9295                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9296                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9297         } else {
9298                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9299         }
9300 }
9301
9302 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9303 {
9304         struct hwrm_port_phy_cfg_input req = {0};
9305
9306         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9307         if (set_pause)
9308                 bnxt_hwrm_set_pause_common(bp, &req);
9309
9310         bnxt_hwrm_set_link_common(bp, &req);
9311
9312         if (set_eee)
9313                 bnxt_hwrm_set_eee(bp, &req);
9314         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9315 }
9316
9317 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9318 {
9319         struct hwrm_port_phy_cfg_input req = {0};
9320
9321         if (!BNXT_SINGLE_PF(bp))
9322                 return 0;
9323
9324         if (pci_num_vf(bp->pdev))
9325                 return 0;
9326
9327         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9328         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9329         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9330 }
9331
9332 static int bnxt_fw_init_one(struct bnxt *bp);
9333
9334 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
9335 {
9336         struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
9337         struct hwrm_func_drv_if_change_input req = {0};
9338         bool resc_reinit = false, fw_reset = false;
9339         u32 flags = 0;
9340         int rc;
9341
9342         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
9343                 return 0;
9344
9345         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
9346         if (up)
9347                 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
9348         mutex_lock(&bp->hwrm_cmd_lock);
9349         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9350         if (!rc)
9351                 flags = le32_to_cpu(resp->flags);
9352         mutex_unlock(&bp->hwrm_cmd_lock);
9353         if (rc)
9354                 return rc;
9355
9356         if (!up)
9357                 return 0;
9358
9359         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
9360                 resc_reinit = true;
9361         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE)
9362                 fw_reset = true;
9363
9364         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
9365                 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
9366                 return -ENODEV;
9367         }
9368         if (resc_reinit || fw_reset) {
9369                 if (fw_reset) {
9370                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9371                                 bnxt_ulp_stop(bp);
9372                         bnxt_free_ctx_mem(bp);
9373                         kfree(bp->ctx);
9374                         bp->ctx = NULL;
9375                         bnxt_dcb_free(bp);
9376                         rc = bnxt_fw_init_one(bp);
9377                         if (rc) {
9378                                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9379                                 return rc;
9380                         }
9381                         bnxt_clear_int_mode(bp);
9382                         rc = bnxt_init_int_mode(bp);
9383                         if (rc) {
9384                                 netdev_err(bp->dev, "init int mode failed\n");
9385                                 return rc;
9386                         }
9387                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
9388                 }
9389                 if (BNXT_NEW_RM(bp)) {
9390                         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9391
9392                         rc = bnxt_hwrm_func_resc_qcaps(bp, true);
9393                         hw_resc->resv_cp_rings = 0;
9394                         hw_resc->resv_stat_ctxs = 0;
9395                         hw_resc->resv_irqs = 0;
9396                         hw_resc->resv_tx_rings = 0;
9397                         hw_resc->resv_rx_rings = 0;
9398                         hw_resc->resv_hw_ring_grps = 0;
9399                         hw_resc->resv_vnics = 0;
9400                         if (!fw_reset) {
9401                                 bp->tx_nr_rings = 0;
9402                                 bp->rx_nr_rings = 0;
9403                         }
9404                 }
9405         }
9406         return 0;
9407 }
9408
9409 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
9410 {
9411         struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
9412         struct hwrm_port_led_qcaps_input req = {0};
9413         struct bnxt_pf_info *pf = &bp->pf;
9414         int rc;
9415
9416         bp->num_leds = 0;
9417         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
9418                 return 0;
9419
9420         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
9421         req.port_id = cpu_to_le16(pf->port_id);
9422         mutex_lock(&bp->hwrm_cmd_lock);
9423         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9424         if (rc) {
9425                 mutex_unlock(&bp->hwrm_cmd_lock);
9426                 return rc;
9427         }
9428         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
9429                 int i;
9430
9431                 bp->num_leds = resp->num_leds;
9432                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
9433                                                  bp->num_leds);
9434                 for (i = 0; i < bp->num_leds; i++) {
9435                         struct bnxt_led_info *led = &bp->leds[i];
9436                         __le16 caps = led->led_state_caps;
9437
9438                         if (!led->led_group_id ||
9439                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
9440                                 bp->num_leds = 0;
9441                                 break;
9442                         }
9443                 }
9444         }
9445         mutex_unlock(&bp->hwrm_cmd_lock);
9446         return 0;
9447 }
9448
9449 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
9450 {
9451         struct hwrm_wol_filter_alloc_input req = {0};
9452         struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
9453         int rc;
9454
9455         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
9456         req.port_id = cpu_to_le16(bp->pf.port_id);
9457         req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
9458         req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
9459         memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
9460         mutex_lock(&bp->hwrm_cmd_lock);
9461         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9462         if (!rc)
9463                 bp->wol_filter_id = resp->wol_filter_id;
9464         mutex_unlock(&bp->hwrm_cmd_lock);
9465         return rc;
9466 }
9467
9468 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
9469 {
9470         struct hwrm_wol_filter_free_input req = {0};
9471
9472         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
9473         req.port_id = cpu_to_le16(bp->pf.port_id);
9474         req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
9475         req.wol_filter_id = bp->wol_filter_id;
9476         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9477 }
9478
9479 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
9480 {
9481         struct hwrm_wol_filter_qcfg_input req = {0};
9482         struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9483         u16 next_handle = 0;
9484         int rc;
9485
9486         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
9487         req.port_id = cpu_to_le16(bp->pf.port_id);
9488         req.handle = cpu_to_le16(handle);
9489         mutex_lock(&bp->hwrm_cmd_lock);
9490         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9491         if (!rc) {
9492                 next_handle = le16_to_cpu(resp->next_handle);
9493                 if (next_handle != 0) {
9494                         if (resp->wol_type ==
9495                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
9496                                 bp->wol = 1;
9497                                 bp->wol_filter_id = resp->wol_filter_id;
9498                         }
9499                 }
9500         }
9501         mutex_unlock(&bp->hwrm_cmd_lock);
9502         return next_handle;
9503 }
9504
9505 static void bnxt_get_wol_settings(struct bnxt *bp)
9506 {
9507         u16 handle = 0;
9508
9509         bp->wol = 0;
9510         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
9511                 return;
9512
9513         do {
9514                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
9515         } while (handle && handle != 0xffff);
9516 }
9517
9518 #ifdef CONFIG_BNXT_HWMON
9519 static ssize_t bnxt_show_temp(struct device *dev,
9520                               struct device_attribute *devattr, char *buf)
9521 {
9522         struct hwrm_temp_monitor_query_input req = {0};
9523         struct hwrm_temp_monitor_query_output *resp;
9524         struct bnxt *bp = dev_get_drvdata(dev);
9525         u32 len = 0;
9526         int rc;
9527
9528         resp = bp->hwrm_cmd_resp_addr;
9529         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9530         mutex_lock(&bp->hwrm_cmd_lock);
9531         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9532         if (!rc)
9533                 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
9534         mutex_unlock(&bp->hwrm_cmd_lock);
9535         return rc ?: len;
9536 }
9537 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
9538
9539 static struct attribute *bnxt_attrs[] = {
9540         &sensor_dev_attr_temp1_input.dev_attr.attr,
9541         NULL
9542 };
9543 ATTRIBUTE_GROUPS(bnxt);
9544
9545 static void bnxt_hwmon_close(struct bnxt *bp)
9546 {
9547         if (bp->hwmon_dev) {
9548                 hwmon_device_unregister(bp->hwmon_dev);
9549                 bp->hwmon_dev = NULL;
9550         }
9551 }
9552
9553 static void bnxt_hwmon_open(struct bnxt *bp)
9554 {
9555         struct hwrm_temp_monitor_query_input req = {0};
9556         struct pci_dev *pdev = bp->pdev;
9557         int rc;
9558
9559         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9560         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9561         if (rc == -EACCES || rc == -EOPNOTSUPP) {
9562                 bnxt_hwmon_close(bp);
9563                 return;
9564         }
9565
9566         if (bp->hwmon_dev)
9567                 return;
9568
9569         bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
9570                                                           DRV_MODULE_NAME, bp,
9571                                                           bnxt_groups);
9572         if (IS_ERR(bp->hwmon_dev)) {
9573                 bp->hwmon_dev = NULL;
9574                 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
9575         }
9576 }
9577 #else
9578 static void bnxt_hwmon_close(struct bnxt *bp)
9579 {
9580 }
9581
9582 static void bnxt_hwmon_open(struct bnxt *bp)
9583 {
9584 }
9585 #endif
9586
9587 static bool bnxt_eee_config_ok(struct bnxt *bp)
9588 {
9589         struct ethtool_eee *eee = &bp->eee;
9590         struct bnxt_link_info *link_info = &bp->link_info;
9591
9592         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
9593                 return true;
9594
9595         if (eee->eee_enabled) {
9596                 u32 advertising =
9597                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
9598
9599                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9600                         eee->eee_enabled = 0;
9601                         return false;
9602                 }
9603                 if (eee->advertised & ~advertising) {
9604                         eee->advertised = advertising & eee->supported;
9605                         return false;
9606                 }
9607         }
9608         return true;
9609 }
9610
9611 static int bnxt_update_phy_setting(struct bnxt *bp)
9612 {
9613         int rc;
9614         bool update_link = false;
9615         bool update_pause = false;
9616         bool update_eee = false;
9617         struct bnxt_link_info *link_info = &bp->link_info;
9618
9619         rc = bnxt_update_link(bp, true);
9620         if (rc) {
9621                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
9622                            rc);
9623                 return rc;
9624         }
9625         if (!BNXT_SINGLE_PF(bp))
9626                 return 0;
9627
9628         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9629             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
9630             link_info->req_flow_ctrl)
9631                 update_pause = true;
9632         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9633             link_info->force_pause_setting != link_info->req_flow_ctrl)
9634                 update_pause = true;
9635         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9636                 if (BNXT_AUTO_MODE(link_info->auto_mode))
9637                         update_link = true;
9638                 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
9639                     link_info->req_link_speed != link_info->force_link_speed)
9640                         update_link = true;
9641                 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
9642                          link_info->req_link_speed != link_info->force_pam4_link_speed)
9643                         update_link = true;
9644                 if (link_info->req_duplex != link_info->duplex_setting)
9645                         update_link = true;
9646         } else {
9647                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
9648                         update_link = true;
9649                 if (link_info->advertising != link_info->auto_link_speeds ||
9650                     link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
9651                         update_link = true;
9652         }
9653
9654         /* The last close may have shutdown the link, so need to call
9655          * PHY_CFG to bring it back up.
9656          */
9657         if (!bp->link_info.link_up)
9658                 update_link = true;
9659
9660         if (!bnxt_eee_config_ok(bp))
9661                 update_eee = true;
9662
9663         if (update_link)
9664                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
9665         else if (update_pause)
9666                 rc = bnxt_hwrm_set_pause(bp);
9667         if (rc) {
9668                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
9669                            rc);
9670                 return rc;
9671         }
9672
9673         return rc;
9674 }
9675
9676 /* Common routine to pre-map certain register block to different GRC window.
9677  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
9678  * in PF and 3 windows in VF that can be customized to map in different
9679  * register blocks.
9680  */
9681 static void bnxt_preset_reg_win(struct bnxt *bp)
9682 {
9683         if (BNXT_PF(bp)) {
9684                 /* CAG registers map to GRC window #4 */
9685                 writel(BNXT_CAG_REG_BASE,
9686                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
9687         }
9688 }
9689
9690 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
9691
9692 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9693 {
9694         int rc = 0;
9695
9696         bnxt_preset_reg_win(bp);
9697         netif_carrier_off(bp->dev);
9698         if (irq_re_init) {
9699                 /* Reserve rings now if none were reserved at driver probe. */
9700                 rc = bnxt_init_dflt_ring_mode(bp);
9701                 if (rc) {
9702                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
9703                         return rc;
9704                 }
9705         }
9706         rc = bnxt_reserve_rings(bp, irq_re_init);
9707         if (rc)
9708                 return rc;
9709         if ((bp->flags & BNXT_FLAG_RFS) &&
9710             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
9711                 /* disable RFS if falling back to INTA */
9712                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
9713                 bp->flags &= ~BNXT_FLAG_RFS;
9714         }
9715
9716         rc = bnxt_alloc_mem(bp, irq_re_init);
9717         if (rc) {
9718                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9719                 goto open_err_free_mem;
9720         }
9721
9722         if (irq_re_init) {
9723                 bnxt_init_napi(bp);
9724                 rc = bnxt_request_irq(bp);
9725                 if (rc) {
9726                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
9727                         goto open_err_irq;
9728                 }
9729         }
9730
9731         rc = bnxt_init_nic(bp, irq_re_init);
9732         if (rc) {
9733                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9734                 goto open_err_irq;
9735         }
9736
9737         bnxt_enable_napi(bp);
9738         bnxt_debug_dev_init(bp);
9739
9740         if (link_re_init) {
9741                 mutex_lock(&bp->link_lock);
9742                 rc = bnxt_update_phy_setting(bp);
9743                 mutex_unlock(&bp->link_lock);
9744                 if (rc) {
9745                         netdev_warn(bp->dev, "failed to update phy settings\n");
9746                         if (BNXT_SINGLE_PF(bp)) {
9747                                 bp->link_info.phy_retry = true;
9748                                 bp->link_info.phy_retry_expires =
9749                                         jiffies + 5 * HZ;
9750                         }
9751                 }
9752         }
9753
9754         if (irq_re_init)
9755                 udp_tunnel_nic_reset_ntf(bp->dev);
9756
9757         set_bit(BNXT_STATE_OPEN, &bp->state);
9758         bnxt_enable_int(bp);
9759         /* Enable TX queues */
9760         bnxt_tx_enable(bp);
9761         mod_timer(&bp->timer, jiffies + bp->current_interval);
9762         /* Poll link status and check for SFP+ module status */
9763         bnxt_get_port_module_status(bp);
9764
9765         /* VF-reps may need to be re-opened after the PF is re-opened */
9766         if (BNXT_PF(bp))
9767                 bnxt_vf_reps_open(bp);
9768         return 0;
9769
9770 open_err_irq:
9771         bnxt_del_napi(bp);
9772
9773 open_err_free_mem:
9774         bnxt_free_skbs(bp);
9775         bnxt_free_irq(bp);
9776         bnxt_free_mem(bp, true);
9777         return rc;
9778 }
9779
9780 /* rtnl_lock held */
9781 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9782 {
9783         int rc = 0;
9784
9785         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
9786                 rc = -EIO;
9787         if (!rc)
9788                 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
9789         if (rc) {
9790                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
9791                 dev_close(bp->dev);
9792         }
9793         return rc;
9794 }
9795
9796 /* rtnl_lock held, open the NIC half way by allocating all resources, but
9797  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
9798  * self tests.
9799  */
9800 int bnxt_half_open_nic(struct bnxt *bp)
9801 {
9802         int rc = 0;
9803
9804         rc = bnxt_alloc_mem(bp, false);
9805         if (rc) {
9806                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9807                 goto half_open_err;
9808         }
9809         rc = bnxt_init_nic(bp, false);
9810         if (rc) {
9811                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9812                 goto half_open_err;
9813         }
9814         return 0;
9815
9816 half_open_err:
9817         bnxt_free_skbs(bp);
9818         bnxt_free_mem(bp, false);
9819         dev_close(bp->dev);
9820         return rc;
9821 }
9822
9823 /* rtnl_lock held, this call can only be made after a previous successful
9824  * call to bnxt_half_open_nic().
9825  */
9826 void bnxt_half_close_nic(struct bnxt *bp)
9827 {
9828         bnxt_hwrm_resource_free(bp, false, false);
9829         bnxt_free_skbs(bp);
9830         bnxt_free_mem(bp, false);
9831 }
9832
9833 static void bnxt_reenable_sriov(struct bnxt *bp)
9834 {
9835         if (BNXT_PF(bp)) {
9836                 struct bnxt_pf_info *pf = &bp->pf;
9837                 int n = pf->active_vfs;
9838
9839                 if (n)
9840                         bnxt_cfg_hw_sriov(bp, &n, true);
9841         }
9842 }
9843
9844 static int bnxt_open(struct net_device *dev)
9845 {
9846         struct bnxt *bp = netdev_priv(dev);
9847         int rc;
9848
9849         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
9850                 netdev_err(bp->dev, "A previous firmware reset did not complete, aborting\n");
9851                 return -ENODEV;
9852         }
9853
9854         rc = bnxt_hwrm_if_change(bp, true);
9855         if (rc)
9856                 return rc;
9857         rc = __bnxt_open_nic(bp, true, true);
9858         if (rc) {
9859                 bnxt_hwrm_if_change(bp, false);
9860         } else {
9861                 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
9862                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9863                                 bnxt_ulp_start(bp, 0);
9864                                 bnxt_reenable_sriov(bp);
9865                         }
9866                 }
9867                 bnxt_hwmon_open(bp);
9868         }
9869
9870         return rc;
9871 }
9872
9873 static bool bnxt_drv_busy(struct bnxt *bp)
9874 {
9875         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
9876                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
9877 }
9878
9879 static void bnxt_get_ring_stats(struct bnxt *bp,
9880                                 struct rtnl_link_stats64 *stats);
9881
9882 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
9883                              bool link_re_init)
9884 {
9885         /* Close the VF-reps before closing PF */
9886         if (BNXT_PF(bp))
9887                 bnxt_vf_reps_close(bp);
9888
9889         /* Change device state to avoid TX queue wake up's */
9890         bnxt_tx_disable(bp);
9891
9892         clear_bit(BNXT_STATE_OPEN, &bp->state);
9893         smp_mb__after_atomic();
9894         while (bnxt_drv_busy(bp))
9895                 msleep(20);
9896
9897         /* Flush rings and and disable interrupts */
9898         bnxt_shutdown_nic(bp, irq_re_init);
9899
9900         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
9901
9902         bnxt_debug_dev_exit(bp);
9903         bnxt_disable_napi(bp);
9904         del_timer_sync(&bp->timer);
9905         bnxt_free_skbs(bp);
9906
9907         /* Save ring stats before shutdown */
9908         if (bp->bnapi && irq_re_init)
9909                 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
9910         if (irq_re_init) {
9911                 bnxt_free_irq(bp);
9912                 bnxt_del_napi(bp);
9913         }
9914         bnxt_free_mem(bp, irq_re_init);
9915 }
9916
9917 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9918 {
9919         int rc = 0;
9920
9921         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9922                 /* If we get here, it means firmware reset is in progress
9923                  * while we are trying to close.  We can safely proceed with
9924                  * the close because we are holding rtnl_lock().  Some firmware
9925                  * messages may fail as we proceed to close.  We set the
9926                  * ABORT_ERR flag here so that the FW reset thread will later
9927                  * abort when it gets the rtnl_lock() and sees the flag.
9928                  */
9929                 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
9930                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9931         }
9932
9933 #ifdef CONFIG_BNXT_SRIOV
9934         if (bp->sriov_cfg) {
9935                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
9936                                                       !bp->sriov_cfg,
9937                                                       BNXT_SRIOV_CFG_WAIT_TMO);
9938                 if (rc)
9939                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
9940         }
9941 #endif
9942         __bnxt_close_nic(bp, irq_re_init, link_re_init);
9943         return rc;
9944 }
9945
9946 static int bnxt_close(struct net_device *dev)
9947 {
9948         struct bnxt *bp = netdev_priv(dev);
9949
9950         bnxt_hwmon_close(bp);
9951         bnxt_close_nic(bp, true, true);
9952         bnxt_hwrm_shutdown_link(bp);
9953         bnxt_hwrm_if_change(bp, false);
9954         return 0;
9955 }
9956
9957 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
9958                                    u16 *val)
9959 {
9960         struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
9961         struct hwrm_port_phy_mdio_read_input req = {0};
9962         int rc;
9963
9964         if (bp->hwrm_spec_code < 0x10a00)
9965                 return -EOPNOTSUPP;
9966
9967         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
9968         req.port_id = cpu_to_le16(bp->pf.port_id);
9969         req.phy_addr = phy_addr;
9970         req.reg_addr = cpu_to_le16(reg & 0x1f);
9971         if (mdio_phy_id_is_c45(phy_addr)) {
9972                 req.cl45_mdio = 1;
9973                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9974                 req.dev_addr = mdio_phy_id_devad(phy_addr);
9975                 req.reg_addr = cpu_to_le16(reg);
9976         }
9977
9978         mutex_lock(&bp->hwrm_cmd_lock);
9979         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9980         if (!rc)
9981                 *val = le16_to_cpu(resp->reg_data);
9982         mutex_unlock(&bp->hwrm_cmd_lock);
9983         return rc;
9984 }
9985
9986 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
9987                                     u16 val)
9988 {
9989         struct hwrm_port_phy_mdio_write_input req = {0};
9990
9991         if (bp->hwrm_spec_code < 0x10a00)
9992                 return -EOPNOTSUPP;
9993
9994         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
9995         req.port_id = cpu_to_le16(bp->pf.port_id);
9996         req.phy_addr = phy_addr;
9997         req.reg_addr = cpu_to_le16(reg & 0x1f);
9998         if (mdio_phy_id_is_c45(phy_addr)) {
9999                 req.cl45_mdio = 1;
10000                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
10001                 req.dev_addr = mdio_phy_id_devad(phy_addr);
10002                 req.reg_addr = cpu_to_le16(reg);
10003         }
10004         req.reg_data = cpu_to_le16(val);
10005
10006         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10007 }
10008
10009 /* rtnl_lock held */
10010 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
10011 {
10012         struct mii_ioctl_data *mdio = if_mii(ifr);
10013         struct bnxt *bp = netdev_priv(dev);
10014         int rc;
10015
10016         switch (cmd) {
10017         case SIOCGMIIPHY:
10018                 mdio->phy_id = bp->link_info.phy_addr;
10019
10020                 fallthrough;
10021         case SIOCGMIIREG: {
10022                 u16 mii_regval = 0;
10023
10024                 if (!netif_running(dev))
10025                         return -EAGAIN;
10026
10027                 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
10028                                              &mii_regval);
10029                 mdio->val_out = mii_regval;
10030                 return rc;
10031         }
10032
10033         case SIOCSMIIREG:
10034                 if (!netif_running(dev))
10035                         return -EAGAIN;
10036
10037                 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
10038                                                 mdio->val_in);
10039
10040         default:
10041                 /* do nothing */
10042                 break;
10043         }
10044         return -EOPNOTSUPP;
10045 }
10046
10047 static void bnxt_get_ring_stats(struct bnxt *bp,
10048                                 struct rtnl_link_stats64 *stats)
10049 {
10050         int i;
10051
10052         for (i = 0; i < bp->cp_nr_rings; i++) {
10053                 struct bnxt_napi *bnapi = bp->bnapi[i];
10054                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10055                 u64 *sw = cpr->stats.sw_stats;
10056
10057                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
10058                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10059                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
10060
10061                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
10062                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
10063                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
10064
10065                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
10066                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
10067                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
10068
10069                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
10070                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
10071                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
10072
10073                 stats->rx_missed_errors +=
10074                         BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
10075
10076                 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10077
10078                 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
10079         }
10080 }
10081
10082 static void bnxt_add_prev_stats(struct bnxt *bp,
10083                                 struct rtnl_link_stats64 *stats)
10084 {
10085         struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
10086
10087         stats->rx_packets += prev_stats->rx_packets;
10088         stats->tx_packets += prev_stats->tx_packets;
10089         stats->rx_bytes += prev_stats->rx_bytes;
10090         stats->tx_bytes += prev_stats->tx_bytes;
10091         stats->rx_missed_errors += prev_stats->rx_missed_errors;
10092         stats->multicast += prev_stats->multicast;
10093         stats->tx_dropped += prev_stats->tx_dropped;
10094 }
10095
10096 static void
10097 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
10098 {
10099         struct bnxt *bp = netdev_priv(dev);
10100
10101         set_bit(BNXT_STATE_READ_STATS, &bp->state);
10102         /* Make sure bnxt_close_nic() sees that we are reading stats before
10103          * we check the BNXT_STATE_OPEN flag.
10104          */
10105         smp_mb__after_atomic();
10106         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10107                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10108                 *stats = bp->net_stats_prev;
10109                 return;
10110         }
10111
10112         bnxt_get_ring_stats(bp, stats);
10113         bnxt_add_prev_stats(bp, stats);
10114
10115         if (bp->flags & BNXT_FLAG_PORT_STATS) {
10116                 u64 *rx = bp->port_stats.sw_stats;
10117                 u64 *tx = bp->port_stats.sw_stats +
10118                           BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10119
10120                 stats->rx_crc_errors =
10121                         BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
10122                 stats->rx_frame_errors =
10123                         BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
10124                 stats->rx_length_errors =
10125                         BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
10126                         BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
10127                         BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
10128                 stats->rx_errors =
10129                         BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
10130                         BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
10131                 stats->collisions =
10132                         BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
10133                 stats->tx_fifo_errors =
10134                         BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
10135                 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
10136         }
10137         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10138 }
10139
10140 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
10141 {
10142         struct net_device *dev = bp->dev;
10143         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10144         struct netdev_hw_addr *ha;
10145         u8 *haddr;
10146         int mc_count = 0;
10147         bool update = false;
10148         int off = 0;
10149
10150         netdev_for_each_mc_addr(ha, dev) {
10151                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
10152                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10153                         vnic->mc_list_count = 0;
10154                         return false;
10155                 }
10156                 haddr = ha->addr;
10157                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
10158                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
10159                         update = true;
10160                 }
10161                 off += ETH_ALEN;
10162                 mc_count++;
10163         }
10164         if (mc_count)
10165                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
10166
10167         if (mc_count != vnic->mc_list_count) {
10168                 vnic->mc_list_count = mc_count;
10169                 update = true;
10170         }
10171         return update;
10172 }
10173
10174 static bool bnxt_uc_list_updated(struct bnxt *bp)
10175 {
10176         struct net_device *dev = bp->dev;
10177         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10178         struct netdev_hw_addr *ha;
10179         int off = 0;
10180
10181         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
10182                 return true;
10183
10184         netdev_for_each_uc_addr(ha, dev) {
10185                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
10186                         return true;
10187
10188                 off += ETH_ALEN;
10189         }
10190         return false;
10191 }
10192
10193 static void bnxt_set_rx_mode(struct net_device *dev)
10194 {
10195         struct bnxt *bp = netdev_priv(dev);
10196         struct bnxt_vnic_info *vnic;
10197         bool mc_update = false;
10198         bool uc_update;
10199         u32 mask;
10200
10201         if (!test_bit(BNXT_STATE_OPEN, &bp->state))
10202                 return;
10203
10204         vnic = &bp->vnic_info[0];
10205         mask = vnic->rx_mask;
10206         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
10207                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
10208                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
10209                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
10210
10211         if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
10212                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10213
10214         uc_update = bnxt_uc_list_updated(bp);
10215
10216         if (dev->flags & IFF_BROADCAST)
10217                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
10218         if (dev->flags & IFF_ALLMULTI) {
10219                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10220                 vnic->mc_list_count = 0;
10221         } else {
10222                 mc_update = bnxt_mc_list_updated(bp, &mask);
10223         }
10224
10225         if (mask != vnic->rx_mask || uc_update || mc_update) {
10226                 vnic->rx_mask = mask;
10227
10228                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
10229                 bnxt_queue_sp_work(bp);
10230         }
10231 }
10232
10233 static int bnxt_cfg_rx_mode(struct bnxt *bp)
10234 {
10235         struct net_device *dev = bp->dev;
10236         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10237         struct netdev_hw_addr *ha;
10238         int i, off = 0, rc;
10239         bool uc_update;
10240
10241         netif_addr_lock_bh(dev);
10242         uc_update = bnxt_uc_list_updated(bp);
10243         netif_addr_unlock_bh(dev);
10244
10245         if (!uc_update)
10246                 goto skip_uc;
10247
10248         mutex_lock(&bp->hwrm_cmd_lock);
10249         for (i = 1; i < vnic->uc_filter_count; i++) {
10250                 struct hwrm_cfa_l2_filter_free_input req = {0};
10251
10252                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
10253                                        -1);
10254
10255                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
10256
10257                 rc = _hwrm_send_message(bp, &req, sizeof(req),
10258                                         HWRM_CMD_TIMEOUT);
10259         }
10260         mutex_unlock(&bp->hwrm_cmd_lock);
10261
10262         vnic->uc_filter_count = 1;
10263
10264         netif_addr_lock_bh(dev);
10265         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
10266                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10267         } else {
10268                 netdev_for_each_uc_addr(ha, dev) {
10269                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
10270                         off += ETH_ALEN;
10271                         vnic->uc_filter_count++;
10272                 }
10273         }
10274         netif_addr_unlock_bh(dev);
10275
10276         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
10277                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
10278                 if (rc) {
10279                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
10280                                    rc);
10281                         vnic->uc_filter_count = i;
10282                         return rc;
10283                 }
10284         }
10285
10286 skip_uc:
10287         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10288         if (rc && vnic->mc_list_count) {
10289                 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
10290                             rc);
10291                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10292                 vnic->mc_list_count = 0;
10293                 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10294         }
10295         if (rc)
10296                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
10297                            rc);
10298
10299         return rc;
10300 }
10301
10302 static bool bnxt_can_reserve_rings(struct bnxt *bp)
10303 {
10304 #ifdef CONFIG_BNXT_SRIOV
10305         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
10306                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10307
10308                 /* No minimum rings were provisioned by the PF.  Don't
10309                  * reserve rings by default when device is down.
10310                  */
10311                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
10312                         return true;
10313
10314                 if (!netif_running(bp->dev))
10315                         return false;
10316         }
10317 #endif
10318         return true;
10319 }
10320
10321 /* If the chip and firmware supports RFS */
10322 static bool bnxt_rfs_supported(struct bnxt *bp)
10323 {
10324         if (bp->flags & BNXT_FLAG_CHIP_P5) {
10325                 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
10326                         return true;
10327                 return false;
10328         }
10329         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
10330                 return true;
10331         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10332                 return true;
10333         return false;
10334 }
10335
10336 /* If runtime conditions support RFS */
10337 static bool bnxt_rfs_capable(struct bnxt *bp)
10338 {
10339 #ifdef CONFIG_RFS_ACCEL
10340         int vnics, max_vnics, max_rss_ctxs;
10341
10342         if (bp->flags & BNXT_FLAG_CHIP_P5)
10343                 return bnxt_rfs_supported(bp);
10344         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
10345                 return false;
10346
10347         vnics = 1 + bp->rx_nr_rings;
10348         max_vnics = bnxt_get_max_func_vnics(bp);
10349         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
10350
10351         /* RSS contexts not a limiting factor */
10352         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10353                 max_rss_ctxs = max_vnics;
10354         if (vnics > max_vnics || vnics > max_rss_ctxs) {
10355                 if (bp->rx_nr_rings > 1)
10356                         netdev_warn(bp->dev,
10357                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
10358                                     min(max_rss_ctxs - 1, max_vnics - 1));
10359                 return false;
10360         }
10361
10362         if (!BNXT_NEW_RM(bp))
10363                 return true;
10364
10365         if (vnics == bp->hw_resc.resv_vnics)
10366                 return true;
10367
10368         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
10369         if (vnics <= bp->hw_resc.resv_vnics)
10370                 return true;
10371
10372         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
10373         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
10374         return false;
10375 #else
10376         return false;
10377 #endif
10378 }
10379
10380 static netdev_features_t bnxt_fix_features(struct net_device *dev,
10381                                            netdev_features_t features)
10382 {
10383         struct bnxt *bp = netdev_priv(dev);
10384         netdev_features_t vlan_features;
10385
10386         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
10387                 features &= ~NETIF_F_NTUPLE;
10388
10389         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10390                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10391
10392         if (!(features & NETIF_F_GRO))
10393                 features &= ~NETIF_F_GRO_HW;
10394
10395         if (features & NETIF_F_GRO_HW)
10396                 features &= ~NETIF_F_LRO;
10397
10398         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
10399          * turned on or off together.
10400          */
10401         vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
10402         if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
10403                 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10404                         features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10405                 else if (vlan_features)
10406                         features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
10407         }
10408 #ifdef CONFIG_BNXT_SRIOV
10409         if (BNXT_VF(bp) && bp->vf.vlan)
10410                 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10411 #endif
10412         return features;
10413 }
10414
10415 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
10416 {
10417         struct bnxt *bp = netdev_priv(dev);
10418         u32 flags = bp->flags;
10419         u32 changes;
10420         int rc = 0;
10421         bool re_init = false;
10422         bool update_tpa = false;
10423
10424         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
10425         if (features & NETIF_F_GRO_HW)
10426                 flags |= BNXT_FLAG_GRO;
10427         else if (features & NETIF_F_LRO)
10428                 flags |= BNXT_FLAG_LRO;
10429
10430         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10431                 flags &= ~BNXT_FLAG_TPA;
10432
10433         if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10434                 flags |= BNXT_FLAG_STRIP_VLAN;
10435
10436         if (features & NETIF_F_NTUPLE)
10437                 flags |= BNXT_FLAG_RFS;
10438
10439         changes = flags ^ bp->flags;
10440         if (changes & BNXT_FLAG_TPA) {
10441                 update_tpa = true;
10442                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
10443                     (flags & BNXT_FLAG_TPA) == 0 ||
10444                     (bp->flags & BNXT_FLAG_CHIP_P5))
10445                         re_init = true;
10446         }
10447
10448         if (changes & ~BNXT_FLAG_TPA)
10449                 re_init = true;
10450
10451         if (flags != bp->flags) {
10452                 u32 old_flags = bp->flags;
10453
10454                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10455                         bp->flags = flags;
10456                         if (update_tpa)
10457                                 bnxt_set_ring_params(bp);
10458                         return rc;
10459                 }
10460
10461                 if (re_init) {
10462                         bnxt_close_nic(bp, false, false);
10463                         bp->flags = flags;
10464                         if (update_tpa)
10465                                 bnxt_set_ring_params(bp);
10466
10467                         return bnxt_open_nic(bp, false, false);
10468                 }
10469                 if (update_tpa) {
10470                         bp->flags = flags;
10471                         rc = bnxt_set_tpa(bp,
10472                                           (flags & BNXT_FLAG_TPA) ?
10473                                           true : false);
10474                         if (rc)
10475                                 bp->flags = old_flags;
10476                 }
10477         }
10478         return rc;
10479 }
10480
10481 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
10482                          u32 *reg_buf)
10483 {
10484         struct hwrm_dbg_read_direct_output *resp = bp->hwrm_cmd_resp_addr;
10485         struct hwrm_dbg_read_direct_input req = {0};
10486         __le32 *dbg_reg_buf;
10487         dma_addr_t mapping;
10488         int rc, i;
10489
10490         dbg_reg_buf = dma_alloc_coherent(&bp->pdev->dev, num_words * 4,
10491                                          &mapping, GFP_KERNEL);
10492         if (!dbg_reg_buf)
10493                 return -ENOMEM;
10494         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_READ_DIRECT, -1, -1);
10495         req.host_dest_addr = cpu_to_le64(mapping);
10496         req.read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
10497         req.read_len32 = cpu_to_le32(num_words);
10498         mutex_lock(&bp->hwrm_cmd_lock);
10499         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10500         if (rc || resp->error_code) {
10501                 rc = -EIO;
10502                 goto dbg_rd_reg_exit;
10503         }
10504         for (i = 0; i < num_words; i++)
10505                 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
10506
10507 dbg_rd_reg_exit:
10508         mutex_unlock(&bp->hwrm_cmd_lock);
10509         dma_free_coherent(&bp->pdev->dev, num_words * 4, dbg_reg_buf, mapping);
10510         return rc;
10511 }
10512
10513 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
10514                                        u32 ring_id, u32 *prod, u32 *cons)
10515 {
10516         struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
10517         struct hwrm_dbg_ring_info_get_input req = {0};
10518         int rc;
10519
10520         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
10521         req.ring_type = ring_type;
10522         req.fw_ring_id = cpu_to_le32(ring_id);
10523         mutex_lock(&bp->hwrm_cmd_lock);
10524         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10525         if (!rc) {
10526                 *prod = le32_to_cpu(resp->producer_index);
10527                 *cons = le32_to_cpu(resp->consumer_index);
10528         }
10529         mutex_unlock(&bp->hwrm_cmd_lock);
10530         return rc;
10531 }
10532
10533 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
10534 {
10535         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
10536         int i = bnapi->index;
10537
10538         if (!txr)
10539                 return;
10540
10541         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
10542                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
10543                     txr->tx_cons);
10544 }
10545
10546 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
10547 {
10548         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
10549         int i = bnapi->index;
10550
10551         if (!rxr)
10552                 return;
10553
10554         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
10555                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
10556                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
10557                     rxr->rx_sw_agg_prod);
10558 }
10559
10560 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
10561 {
10562         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10563         int i = bnapi->index;
10564
10565         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
10566                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
10567 }
10568
10569 static void bnxt_dbg_dump_states(struct bnxt *bp)
10570 {
10571         int i;
10572         struct bnxt_napi *bnapi;
10573
10574         for (i = 0; i < bp->cp_nr_rings; i++) {
10575                 bnapi = bp->bnapi[i];
10576                 if (netif_msg_drv(bp)) {
10577                         bnxt_dump_tx_sw_state(bnapi);
10578                         bnxt_dump_rx_sw_state(bnapi);
10579                         bnxt_dump_cp_sw_state(bnapi);
10580                 }
10581         }
10582 }
10583
10584 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
10585 {
10586         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
10587         struct hwrm_ring_reset_input req = {0};
10588         struct bnxt_napi *bnapi = rxr->bnapi;
10589         struct bnxt_cp_ring_info *cpr;
10590         u16 cp_ring_id;
10591
10592         cpr = &bnapi->cp_ring;
10593         cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
10594         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_RESET, cp_ring_id, -1);
10595         req.ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
10596         req.ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
10597         return hwrm_send_message_silent(bp, &req, sizeof(req),
10598                                         HWRM_CMD_TIMEOUT);
10599 }
10600
10601 static void bnxt_reset_task(struct bnxt *bp, bool silent)
10602 {
10603         if (!silent)
10604                 bnxt_dbg_dump_states(bp);
10605         if (netif_running(bp->dev)) {
10606                 int rc;
10607
10608                 if (silent) {
10609                         bnxt_close_nic(bp, false, false);
10610                         bnxt_open_nic(bp, false, false);
10611                 } else {
10612                         bnxt_ulp_stop(bp);
10613                         bnxt_close_nic(bp, true, false);
10614                         rc = bnxt_open_nic(bp, true, false);
10615                         bnxt_ulp_start(bp, rc);
10616                 }
10617         }
10618 }
10619
10620 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
10621 {
10622         struct bnxt *bp = netdev_priv(dev);
10623
10624         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
10625         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
10626         bnxt_queue_sp_work(bp);
10627 }
10628
10629 static void bnxt_fw_health_check(struct bnxt *bp)
10630 {
10631         struct bnxt_fw_health *fw_health = bp->fw_health;
10632         u32 val;
10633
10634         if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10635                 return;
10636
10637         if (fw_health->tmr_counter) {
10638                 fw_health->tmr_counter--;
10639                 return;
10640         }
10641
10642         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10643         if (val == fw_health->last_fw_heartbeat)
10644                 goto fw_reset;
10645
10646         fw_health->last_fw_heartbeat = val;
10647
10648         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10649         if (val != fw_health->last_fw_reset_cnt)
10650                 goto fw_reset;
10651
10652         fw_health->tmr_counter = fw_health->tmr_multiplier;
10653         return;
10654
10655 fw_reset:
10656         set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
10657         bnxt_queue_sp_work(bp);
10658 }
10659
10660 static void bnxt_timer(struct timer_list *t)
10661 {
10662         struct bnxt *bp = from_timer(bp, t, timer);
10663         struct net_device *dev = bp->dev;
10664
10665         if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
10666                 return;
10667
10668         if (atomic_read(&bp->intr_sem) != 0)
10669                 goto bnxt_restart_timer;
10670
10671         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
10672                 bnxt_fw_health_check(bp);
10673
10674         if (bp->link_info.link_up && bp->stats_coal_ticks) {
10675                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
10676                 bnxt_queue_sp_work(bp);
10677         }
10678
10679         if (bnxt_tc_flower_enabled(bp)) {
10680                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
10681                 bnxt_queue_sp_work(bp);
10682         }
10683
10684 #ifdef CONFIG_RFS_ACCEL
10685         if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) {
10686                 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
10687                 bnxt_queue_sp_work(bp);
10688         }
10689 #endif /*CONFIG_RFS_ACCEL*/
10690
10691         if (bp->link_info.phy_retry) {
10692                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
10693                         bp->link_info.phy_retry = false;
10694                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
10695                 } else {
10696                         set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
10697                         bnxt_queue_sp_work(bp);
10698                 }
10699         }
10700
10701         if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
10702             netif_carrier_ok(dev)) {
10703                 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
10704                 bnxt_queue_sp_work(bp);
10705         }
10706 bnxt_restart_timer:
10707         mod_timer(&bp->timer, jiffies + bp->current_interval);
10708 }
10709
10710 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
10711 {
10712         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
10713          * set.  If the device is being closed, bnxt_close() may be holding
10714          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
10715          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
10716          */
10717         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10718         rtnl_lock();
10719 }
10720
10721 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
10722 {
10723         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10724         rtnl_unlock();
10725 }
10726
10727 /* Only called from bnxt_sp_task() */
10728 static void bnxt_reset(struct bnxt *bp, bool silent)
10729 {
10730         bnxt_rtnl_lock_sp(bp);
10731         if (test_bit(BNXT_STATE_OPEN, &bp->state))
10732                 bnxt_reset_task(bp, silent);
10733         bnxt_rtnl_unlock_sp(bp);
10734 }
10735
10736 /* Only called from bnxt_sp_task() */
10737 static void bnxt_rx_ring_reset(struct bnxt *bp)
10738 {
10739         int i;
10740
10741         bnxt_rtnl_lock_sp(bp);
10742         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10743                 bnxt_rtnl_unlock_sp(bp);
10744                 return;
10745         }
10746         /* Disable and flush TPA before resetting the RX ring */
10747         if (bp->flags & BNXT_FLAG_TPA)
10748                 bnxt_set_tpa(bp, false);
10749         for (i = 0; i < bp->rx_nr_rings; i++) {
10750                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
10751                 struct bnxt_cp_ring_info *cpr;
10752                 int rc;
10753
10754                 if (!rxr->bnapi->in_reset)
10755                         continue;
10756
10757                 rc = bnxt_hwrm_rx_ring_reset(bp, i);
10758                 if (rc) {
10759                         if (rc == -EINVAL || rc == -EOPNOTSUPP)
10760                                 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
10761                         else
10762                                 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
10763                                             rc);
10764                         bnxt_reset_task(bp, true);
10765                         break;
10766                 }
10767                 bnxt_free_one_rx_ring_skbs(bp, i);
10768                 rxr->rx_prod = 0;
10769                 rxr->rx_agg_prod = 0;
10770                 rxr->rx_sw_agg_prod = 0;
10771                 rxr->rx_next_cons = 0;
10772                 rxr->bnapi->in_reset = false;
10773                 bnxt_alloc_one_rx_ring(bp, i);
10774                 cpr = &rxr->bnapi->cp_ring;
10775                 cpr->sw_stats.rx.rx_resets++;
10776                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
10777                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
10778                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
10779         }
10780         if (bp->flags & BNXT_FLAG_TPA)
10781                 bnxt_set_tpa(bp, true);
10782         bnxt_rtnl_unlock_sp(bp);
10783 }
10784
10785 static void bnxt_fw_reset_close(struct bnxt *bp)
10786 {
10787         bnxt_ulp_stop(bp);
10788         /* When firmware is fatal state, disable PCI device to prevent
10789          * any potential bad DMAs before freeing kernel memory.
10790          */
10791         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10792                 pci_disable_device(bp->pdev);
10793         __bnxt_close_nic(bp, true, false);
10794         bnxt_clear_int_mode(bp);
10795         bnxt_hwrm_func_drv_unrgtr(bp);
10796         if (pci_is_enabled(bp->pdev))
10797                 pci_disable_device(bp->pdev);
10798         bnxt_free_ctx_mem(bp);
10799         kfree(bp->ctx);
10800         bp->ctx = NULL;
10801 }
10802
10803 static bool is_bnxt_fw_ok(struct bnxt *bp)
10804 {
10805         struct bnxt_fw_health *fw_health = bp->fw_health;
10806         bool no_heartbeat = false, has_reset = false;
10807         u32 val;
10808
10809         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10810         if (val == fw_health->last_fw_heartbeat)
10811                 no_heartbeat = true;
10812
10813         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10814         if (val != fw_health->last_fw_reset_cnt)
10815                 has_reset = true;
10816
10817         if (!no_heartbeat && has_reset)
10818                 return true;
10819
10820         return false;
10821 }
10822
10823 /* rtnl_lock is acquired before calling this function */
10824 static void bnxt_force_fw_reset(struct bnxt *bp)
10825 {
10826         struct bnxt_fw_health *fw_health = bp->fw_health;
10827         u32 wait_dsecs;
10828
10829         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
10830             test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10831                 return;
10832
10833         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10834         bnxt_fw_reset_close(bp);
10835         wait_dsecs = fw_health->master_func_wait_dsecs;
10836         if (fw_health->master) {
10837                 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
10838                         wait_dsecs = 0;
10839                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
10840         } else {
10841                 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
10842                 wait_dsecs = fw_health->normal_func_wait_dsecs;
10843                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10844         }
10845
10846         bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
10847         bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
10848         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
10849 }
10850
10851 void bnxt_fw_exception(struct bnxt *bp)
10852 {
10853         netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
10854         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
10855         bnxt_rtnl_lock_sp(bp);
10856         bnxt_force_fw_reset(bp);
10857         bnxt_rtnl_unlock_sp(bp);
10858 }
10859
10860 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
10861  * < 0 on error.
10862  */
10863 static int bnxt_get_registered_vfs(struct bnxt *bp)
10864 {
10865 #ifdef CONFIG_BNXT_SRIOV
10866         int rc;
10867
10868         if (!BNXT_PF(bp))
10869                 return 0;
10870
10871         rc = bnxt_hwrm_func_qcfg(bp);
10872         if (rc) {
10873                 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
10874                 return rc;
10875         }
10876         if (bp->pf.registered_vfs)
10877                 return bp->pf.registered_vfs;
10878         if (bp->sriov_cfg)
10879                 return 1;
10880 #endif
10881         return 0;
10882 }
10883
10884 void bnxt_fw_reset(struct bnxt *bp)
10885 {
10886         bnxt_rtnl_lock_sp(bp);
10887         if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
10888             !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10889                 int n = 0, tmo;
10890
10891                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10892                 if (bp->pf.active_vfs &&
10893                     !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10894                         n = bnxt_get_registered_vfs(bp);
10895                 if (n < 0) {
10896                         netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
10897                                    n);
10898                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10899                         dev_close(bp->dev);
10900                         goto fw_reset_exit;
10901                 } else if (n > 0) {
10902                         u16 vf_tmo_dsecs = n * 10;
10903
10904                         if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
10905                                 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
10906                         bp->fw_reset_state =
10907                                 BNXT_FW_RESET_STATE_POLL_VF;
10908                         bnxt_queue_fw_reset_work(bp, HZ / 10);
10909                         goto fw_reset_exit;
10910                 }
10911                 bnxt_fw_reset_close(bp);
10912                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
10913                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
10914                         tmo = HZ / 10;
10915                 } else {
10916                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10917                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
10918                 }
10919                 bnxt_queue_fw_reset_work(bp, tmo);
10920         }
10921 fw_reset_exit:
10922         bnxt_rtnl_unlock_sp(bp);
10923 }
10924
10925 static void bnxt_chk_missed_irq(struct bnxt *bp)
10926 {
10927         int i;
10928
10929         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
10930                 return;
10931
10932         for (i = 0; i < bp->cp_nr_rings; i++) {
10933                 struct bnxt_napi *bnapi = bp->bnapi[i];
10934                 struct bnxt_cp_ring_info *cpr;
10935                 u32 fw_ring_id;
10936                 int j;
10937
10938                 if (!bnapi)
10939                         continue;
10940
10941                 cpr = &bnapi->cp_ring;
10942                 for (j = 0; j < 2; j++) {
10943                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
10944                         u32 val[2];
10945
10946                         if (!cpr2 || cpr2->has_more_work ||
10947                             !bnxt_has_work(bp, cpr2))
10948                                 continue;
10949
10950                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
10951                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
10952                                 continue;
10953                         }
10954                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
10955                         bnxt_dbg_hwrm_ring_info_get(bp,
10956                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
10957                                 fw_ring_id, &val[0], &val[1]);
10958                         cpr->sw_stats.cmn.missed_irqs++;
10959                 }
10960         }
10961 }
10962
10963 static void bnxt_cfg_ntp_filters(struct bnxt *);
10964
10965 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
10966 {
10967         struct bnxt_link_info *link_info = &bp->link_info;
10968
10969         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
10970                 link_info->autoneg = BNXT_AUTONEG_SPEED;
10971                 if (bp->hwrm_spec_code >= 0x10201) {
10972                         if (link_info->auto_pause_setting &
10973                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
10974                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10975                 } else {
10976                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10977                 }
10978                 link_info->advertising = link_info->auto_link_speeds;
10979                 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
10980         } else {
10981                 link_info->req_link_speed = link_info->force_link_speed;
10982                 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
10983                 if (link_info->force_pam4_link_speed) {
10984                         link_info->req_link_speed =
10985                                 link_info->force_pam4_link_speed;
10986                         link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
10987                 }
10988                 link_info->req_duplex = link_info->duplex_setting;
10989         }
10990         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
10991                 link_info->req_flow_ctrl =
10992                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
10993         else
10994                 link_info->req_flow_ctrl = link_info->force_pause_setting;
10995 }
10996
10997 static void bnxt_sp_task(struct work_struct *work)
10998 {
10999         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
11000
11001         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11002         smp_mb__after_atomic();
11003         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11004                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11005                 return;
11006         }
11007
11008         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
11009                 bnxt_cfg_rx_mode(bp);
11010
11011         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
11012                 bnxt_cfg_ntp_filters(bp);
11013         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
11014                 bnxt_hwrm_exec_fwd_req(bp);
11015         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
11016                 bnxt_hwrm_port_qstats(bp, 0);
11017                 bnxt_hwrm_port_qstats_ext(bp, 0);
11018                 bnxt_accumulate_all_stats(bp);
11019         }
11020
11021         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
11022                 int rc;
11023
11024                 mutex_lock(&bp->link_lock);
11025                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
11026                                        &bp->sp_event))
11027                         bnxt_hwrm_phy_qcaps(bp);
11028
11029                 rc = bnxt_update_link(bp, true);
11030                 if (rc)
11031                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
11032                                    rc);
11033
11034                 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
11035                                        &bp->sp_event))
11036                         bnxt_init_ethtool_link_settings(bp);
11037                 mutex_unlock(&bp->link_lock);
11038         }
11039         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
11040                 int rc;
11041
11042                 mutex_lock(&bp->link_lock);
11043                 rc = bnxt_update_phy_setting(bp);
11044                 mutex_unlock(&bp->link_lock);
11045                 if (rc) {
11046                         netdev_warn(bp->dev, "update phy settings retry failed\n");
11047                 } else {
11048                         bp->link_info.phy_retry = false;
11049                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
11050                 }
11051         }
11052         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
11053                 mutex_lock(&bp->link_lock);
11054                 bnxt_get_port_module_status(bp);
11055                 mutex_unlock(&bp->link_lock);
11056         }
11057
11058         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
11059                 bnxt_tc_flow_stats_work(bp);
11060
11061         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
11062                 bnxt_chk_missed_irq(bp);
11063
11064         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
11065          * must be the last functions to be called before exiting.
11066          */
11067         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
11068                 bnxt_reset(bp, false);
11069
11070         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
11071                 bnxt_reset(bp, true);
11072
11073         if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
11074                 bnxt_rx_ring_reset(bp);
11075
11076         if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event))
11077                 bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT);
11078
11079         if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
11080                 if (!is_bnxt_fw_ok(bp))
11081                         bnxt_devlink_health_report(bp,
11082                                                    BNXT_FW_EXCEPTION_SP_EVENT);
11083         }
11084
11085         smp_mb__before_atomic();
11086         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11087 }
11088
11089 /* Under rtnl_lock */
11090 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
11091                      int tx_xdp)
11092 {
11093         int max_rx, max_tx, tx_sets = 1;
11094         int tx_rings_needed, stats;
11095         int rx_rings = rx;
11096         int cp, vnics, rc;
11097
11098         if (tcs)
11099                 tx_sets = tcs;
11100
11101         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
11102         if (rc)
11103                 return rc;
11104
11105         if (max_rx < rx)
11106                 return -ENOMEM;
11107
11108         tx_rings_needed = tx * tx_sets + tx_xdp;
11109         if (max_tx < tx_rings_needed)
11110                 return -ENOMEM;
11111
11112         vnics = 1;
11113         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
11114                 vnics += rx_rings;
11115
11116         if (bp->flags & BNXT_FLAG_AGG_RINGS)
11117                 rx_rings <<= 1;
11118         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
11119         stats = cp;
11120         if (BNXT_NEW_RM(bp)) {
11121                 cp += bnxt_get_ulp_msix_num(bp);
11122                 stats += bnxt_get_ulp_stat_ctxs(bp);
11123         }
11124         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
11125                                      stats, vnics);
11126 }
11127
11128 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
11129 {
11130         if (bp->bar2) {
11131                 pci_iounmap(pdev, bp->bar2);
11132                 bp->bar2 = NULL;
11133         }
11134
11135         if (bp->bar1) {
11136                 pci_iounmap(pdev, bp->bar1);
11137                 bp->bar1 = NULL;
11138         }
11139
11140         if (bp->bar0) {
11141                 pci_iounmap(pdev, bp->bar0);
11142                 bp->bar0 = NULL;
11143         }
11144 }
11145
11146 static void bnxt_cleanup_pci(struct bnxt *bp)
11147 {
11148         bnxt_unmap_bars(bp, bp->pdev);
11149         pci_release_regions(bp->pdev);
11150         if (pci_is_enabled(bp->pdev))
11151                 pci_disable_device(bp->pdev);
11152 }
11153
11154 static void bnxt_init_dflt_coal(struct bnxt *bp)
11155 {
11156         struct bnxt_coal *coal;
11157
11158         /* Tick values in micro seconds.
11159          * 1 coal_buf x bufs_per_record = 1 completion record.
11160          */
11161         coal = &bp->rx_coal;
11162         coal->coal_ticks = 10;
11163         coal->coal_bufs = 30;
11164         coal->coal_ticks_irq = 1;
11165         coal->coal_bufs_irq = 2;
11166         coal->idle_thresh = 50;
11167         coal->bufs_per_record = 2;
11168         coal->budget = 64;              /* NAPI budget */
11169
11170         coal = &bp->tx_coal;
11171         coal->coal_ticks = 28;
11172         coal->coal_bufs = 30;
11173         coal->coal_ticks_irq = 2;
11174         coal->coal_bufs_irq = 2;
11175         coal->bufs_per_record = 1;
11176
11177         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
11178 }
11179
11180 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
11181 {
11182 #ifdef CONFIG_TEE_BNXT_FW
11183         int rc = tee_bnxt_fw_load();
11184
11185         if (rc)
11186                 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
11187
11188         return rc;
11189 #else
11190         netdev_err(bp->dev, "OP-TEE not supported\n");
11191         return -ENODEV;
11192 #endif
11193 }
11194
11195 static int bnxt_fw_init_one_p1(struct bnxt *bp)
11196 {
11197         int rc;
11198
11199         bp->fw_cap = 0;
11200         rc = bnxt_hwrm_ver_get(bp);
11201         bnxt_try_map_fw_health_reg(bp);
11202         if (rc) {
11203                 if (bp->fw_health && bp->fw_health->status_reliable) {
11204                         u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11205
11206                         netdev_err(bp->dev,
11207                                    "Firmware not responding, status: 0x%x\n",
11208                                    sts);
11209                         if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
11210                                 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
11211                                 rc = bnxt_fw_reset_via_optee(bp);
11212                                 if (!rc)
11213                                         rc = bnxt_hwrm_ver_get(bp);
11214                         }
11215                 }
11216                 if (rc)
11217                         return rc;
11218         }
11219
11220         if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
11221                 rc = bnxt_alloc_kong_hwrm_resources(bp);
11222                 if (rc)
11223                         bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
11224         }
11225
11226         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
11227             bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
11228                 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
11229                 if (rc)
11230                         return rc;
11231         }
11232         bnxt_nvm_cfg_ver_get(bp);
11233
11234         rc = bnxt_hwrm_func_reset(bp);
11235         if (rc)
11236                 return -ENODEV;
11237
11238         bnxt_hwrm_fw_set_time(bp);
11239         return 0;
11240 }
11241
11242 static int bnxt_fw_init_one_p2(struct bnxt *bp)
11243 {
11244         int rc;
11245
11246         /* Get the MAX capabilities for this function */
11247         rc = bnxt_hwrm_func_qcaps(bp);
11248         if (rc) {
11249                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
11250                            rc);
11251                 return -ENODEV;
11252         }
11253
11254         rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
11255         if (rc)
11256                 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
11257                             rc);
11258
11259         if (bnxt_alloc_fw_health(bp)) {
11260                 netdev_warn(bp->dev, "no memory for firmware error recovery\n");
11261         } else {
11262                 rc = bnxt_hwrm_error_recovery_qcfg(bp);
11263                 if (rc)
11264                         netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
11265                                     rc);
11266         }
11267
11268         rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
11269         if (rc)
11270                 return -ENODEV;
11271
11272         bnxt_hwrm_func_qcfg(bp);
11273         bnxt_hwrm_vnic_qcaps(bp);
11274         bnxt_hwrm_port_led_qcaps(bp);
11275         bnxt_ethtool_init(bp);
11276         bnxt_dcb_init(bp);
11277         return 0;
11278 }
11279
11280 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
11281 {
11282         bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
11283         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
11284                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
11285                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
11286                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
11287         if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
11288                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
11289                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
11290                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
11291         }
11292 }
11293
11294 static void bnxt_set_dflt_rfs(struct bnxt *bp)
11295 {
11296         struct net_device *dev = bp->dev;
11297
11298         dev->hw_features &= ~NETIF_F_NTUPLE;
11299         dev->features &= ~NETIF_F_NTUPLE;
11300         bp->flags &= ~BNXT_FLAG_RFS;
11301         if (bnxt_rfs_supported(bp)) {
11302                 dev->hw_features |= NETIF_F_NTUPLE;
11303                 if (bnxt_rfs_capable(bp)) {
11304                         bp->flags |= BNXT_FLAG_RFS;
11305                         dev->features |= NETIF_F_NTUPLE;
11306                 }
11307         }
11308 }
11309
11310 static void bnxt_fw_init_one_p3(struct bnxt *bp)
11311 {
11312         struct pci_dev *pdev = bp->pdev;
11313
11314         bnxt_set_dflt_rss_hash_type(bp);
11315         bnxt_set_dflt_rfs(bp);
11316
11317         bnxt_get_wol_settings(bp);
11318         if (bp->flags & BNXT_FLAG_WOL_CAP)
11319                 device_set_wakeup_enable(&pdev->dev, bp->wol);
11320         else
11321                 device_set_wakeup_capable(&pdev->dev, false);
11322
11323         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
11324         bnxt_hwrm_coal_params_qcaps(bp);
11325 }
11326
11327 static int bnxt_fw_init_one(struct bnxt *bp)
11328 {
11329         int rc;
11330
11331         rc = bnxt_fw_init_one_p1(bp);
11332         if (rc) {
11333                 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
11334                 return rc;
11335         }
11336         rc = bnxt_fw_init_one_p2(bp);
11337         if (rc) {
11338                 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
11339                 return rc;
11340         }
11341         rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
11342         if (rc)
11343                 return rc;
11344
11345         /* In case fw capabilities have changed, destroy the unneeded
11346          * reporters and create newly capable ones.
11347          */
11348         bnxt_dl_fw_reporters_destroy(bp, false);
11349         bnxt_dl_fw_reporters_create(bp);
11350         bnxt_fw_init_one_p3(bp);
11351         return 0;
11352 }
11353
11354 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
11355 {
11356         struct bnxt_fw_health *fw_health = bp->fw_health;
11357         u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
11358         u32 val = fw_health->fw_reset_seq_vals[reg_idx];
11359         u32 reg_type, reg_off, delay_msecs;
11360
11361         delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
11362         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
11363         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
11364         switch (reg_type) {
11365         case BNXT_FW_HEALTH_REG_TYPE_CFG:
11366                 pci_write_config_dword(bp->pdev, reg_off, val);
11367                 break;
11368         case BNXT_FW_HEALTH_REG_TYPE_GRC:
11369                 writel(reg_off & BNXT_GRC_BASE_MASK,
11370                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
11371                 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
11372                 fallthrough;
11373         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
11374                 writel(val, bp->bar0 + reg_off);
11375                 break;
11376         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
11377                 writel(val, bp->bar1 + reg_off);
11378                 break;
11379         }
11380         if (delay_msecs) {
11381                 pci_read_config_dword(bp->pdev, 0, &val);
11382                 msleep(delay_msecs);
11383         }
11384 }
11385
11386 static void bnxt_reset_all(struct bnxt *bp)
11387 {
11388         struct bnxt_fw_health *fw_health = bp->fw_health;
11389         int i, rc;
11390
11391         if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11392                 bnxt_fw_reset_via_optee(bp);
11393                 bp->fw_reset_timestamp = jiffies;
11394                 return;
11395         }
11396
11397         if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
11398                 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
11399                         bnxt_fw_reset_writel(bp, i);
11400         } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
11401                 struct hwrm_fw_reset_input req = {0};
11402
11403                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1);
11404                 req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
11405                 req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
11406                 req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
11407                 req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
11408                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
11409                 if (rc)
11410                         netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
11411         }
11412         bp->fw_reset_timestamp = jiffies;
11413 }
11414
11415 static void bnxt_fw_reset_task(struct work_struct *work)
11416 {
11417         struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
11418         int rc;
11419
11420         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11421                 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
11422                 return;
11423         }
11424
11425         switch (bp->fw_reset_state) {
11426         case BNXT_FW_RESET_STATE_POLL_VF: {
11427                 int n = bnxt_get_registered_vfs(bp);
11428                 int tmo;
11429
11430                 if (n < 0) {
11431                         netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
11432                                    n, jiffies_to_msecs(jiffies -
11433                                    bp->fw_reset_timestamp));
11434                         goto fw_reset_abort;
11435                 } else if (n > 0) {
11436                         if (time_after(jiffies, bp->fw_reset_timestamp +
11437                                        (bp->fw_reset_max_dsecs * HZ / 10))) {
11438                                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11439                                 bp->fw_reset_state = 0;
11440                                 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
11441                                            n);
11442                                 return;
11443                         }
11444                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11445                         return;
11446                 }
11447                 bp->fw_reset_timestamp = jiffies;
11448                 rtnl_lock();
11449                 bnxt_fw_reset_close(bp);
11450                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11451                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11452                         tmo = HZ / 10;
11453                 } else {
11454                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11455                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
11456                 }
11457                 rtnl_unlock();
11458                 bnxt_queue_fw_reset_work(bp, tmo);
11459                 return;
11460         }
11461         case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
11462                 u32 val;
11463
11464                 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11465                 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
11466                     !time_after(jiffies, bp->fw_reset_timestamp +
11467                     (bp->fw_reset_max_dsecs * HZ / 10))) {
11468                         bnxt_queue_fw_reset_work(bp, HZ / 5);
11469                         return;
11470                 }
11471
11472                 if (!bp->fw_health->master) {
11473                         u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
11474
11475                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11476                         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11477                         return;
11478                 }
11479                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11480         }
11481                 fallthrough;
11482         case BNXT_FW_RESET_STATE_RESET_FW:
11483                 bnxt_reset_all(bp);
11484                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11485                 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
11486                 return;
11487         case BNXT_FW_RESET_STATE_ENABLE_DEV:
11488                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11489                         u32 val;
11490
11491                         val = bnxt_fw_health_readl(bp,
11492                                                    BNXT_FW_RESET_INPROG_REG);
11493                         if (val)
11494                                 netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n",
11495                                             val);
11496                 }
11497                 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11498                 if (pci_enable_device(bp->pdev)) {
11499                         netdev_err(bp->dev, "Cannot re-enable PCI device\n");
11500                         goto fw_reset_abort;
11501                 }
11502                 pci_set_master(bp->pdev);
11503                 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
11504                 fallthrough;
11505         case BNXT_FW_RESET_STATE_POLL_FW:
11506                 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
11507                 rc = __bnxt_hwrm_ver_get(bp, true);
11508                 if (rc) {
11509                         if (time_after(jiffies, bp->fw_reset_timestamp +
11510                                        (bp->fw_reset_max_dsecs * HZ / 10))) {
11511                                 netdev_err(bp->dev, "Firmware reset aborted\n");
11512                                 goto fw_reset_abort_status;
11513                         }
11514                         bnxt_queue_fw_reset_work(bp, HZ / 5);
11515                         return;
11516                 }
11517                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
11518                 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
11519                 fallthrough;
11520         case BNXT_FW_RESET_STATE_OPENING:
11521                 while (!rtnl_trylock()) {
11522                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11523                         return;
11524                 }
11525                 rc = bnxt_open(bp->dev);
11526                 if (rc) {
11527                         netdev_err(bp->dev, "bnxt_open_nic() failed\n");
11528                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11529                         dev_close(bp->dev);
11530                 }
11531
11532                 bp->fw_reset_state = 0;
11533                 /* Make sure fw_reset_state is 0 before clearing the flag */
11534                 smp_mb__before_atomic();
11535                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11536                 bnxt_ulp_start(bp, rc);
11537                 if (!rc)
11538                         bnxt_reenable_sriov(bp);
11539                 bnxt_dl_health_recovery_done(bp);
11540                 bnxt_dl_health_status_update(bp, true);
11541                 rtnl_unlock();
11542                 break;
11543         }
11544         return;
11545
11546 fw_reset_abort_status:
11547         if (bp->fw_health->status_reliable ||
11548             (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
11549                 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11550
11551                 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
11552         }
11553 fw_reset_abort:
11554         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11555         if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF)
11556                 bnxt_dl_health_status_update(bp, false);
11557         bp->fw_reset_state = 0;
11558         rtnl_lock();
11559         dev_close(bp->dev);
11560         rtnl_unlock();
11561 }
11562
11563 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
11564 {
11565         int rc;
11566         struct bnxt *bp = netdev_priv(dev);
11567
11568         SET_NETDEV_DEV(dev, &pdev->dev);
11569
11570         /* enable device (incl. PCI PM wakeup), and bus-mastering */
11571         rc = pci_enable_device(pdev);
11572         if (rc) {
11573                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
11574                 goto init_err;
11575         }
11576
11577         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
11578                 dev_err(&pdev->dev,
11579                         "Cannot find PCI device base address, aborting\n");
11580                 rc = -ENODEV;
11581                 goto init_err_disable;
11582         }
11583
11584         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
11585         if (rc) {
11586                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
11587                 goto init_err_disable;
11588         }
11589
11590         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
11591             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
11592                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
11593                 rc = -EIO;
11594                 goto init_err_release;
11595         }
11596
11597         pci_set_master(pdev);
11598
11599         bp->dev = dev;
11600         bp->pdev = pdev;
11601
11602         /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
11603          * determines the BAR size.
11604          */
11605         bp->bar0 = pci_ioremap_bar(pdev, 0);
11606         if (!bp->bar0) {
11607                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
11608                 rc = -ENOMEM;
11609                 goto init_err_release;
11610         }
11611
11612         bp->bar2 = pci_ioremap_bar(pdev, 4);
11613         if (!bp->bar2) {
11614                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
11615                 rc = -ENOMEM;
11616                 goto init_err_release;
11617         }
11618
11619         pci_enable_pcie_error_reporting(pdev);
11620
11621         INIT_WORK(&bp->sp_task, bnxt_sp_task);
11622         INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
11623
11624         spin_lock_init(&bp->ntp_fltr_lock);
11625 #if BITS_PER_LONG == 32
11626         spin_lock_init(&bp->db_lock);
11627 #endif
11628
11629         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
11630         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
11631
11632         bnxt_init_dflt_coal(bp);
11633
11634         timer_setup(&bp->timer, bnxt_timer, 0);
11635         bp->current_interval = BNXT_TIMER_INTERVAL;
11636
11637         bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
11638         bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
11639
11640         clear_bit(BNXT_STATE_OPEN, &bp->state);
11641         return 0;
11642
11643 init_err_release:
11644         bnxt_unmap_bars(bp, pdev);
11645         pci_release_regions(pdev);
11646
11647 init_err_disable:
11648         pci_disable_device(pdev);
11649
11650 init_err:
11651         return rc;
11652 }
11653
11654 /* rtnl_lock held */
11655 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
11656 {
11657         struct sockaddr *addr = p;
11658         struct bnxt *bp = netdev_priv(dev);
11659         int rc = 0;
11660
11661         if (!is_valid_ether_addr(addr->sa_data))
11662                 return -EADDRNOTAVAIL;
11663
11664         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
11665                 return 0;
11666
11667         rc = bnxt_approve_mac(bp, addr->sa_data, true);
11668         if (rc)
11669                 return rc;
11670
11671         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
11672         if (netif_running(dev)) {
11673                 bnxt_close_nic(bp, false, false);
11674                 rc = bnxt_open_nic(bp, false, false);
11675         }
11676
11677         return rc;
11678 }
11679
11680 /* rtnl_lock held */
11681 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
11682 {
11683         struct bnxt *bp = netdev_priv(dev);
11684
11685         if (netif_running(dev))
11686                 bnxt_close_nic(bp, true, false);
11687
11688         dev->mtu = new_mtu;
11689         bnxt_set_ring_params(bp);
11690
11691         if (netif_running(dev))
11692                 return bnxt_open_nic(bp, true, false);
11693
11694         return 0;
11695 }
11696
11697 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
11698 {
11699         struct bnxt *bp = netdev_priv(dev);
11700         bool sh = false;
11701         int rc;
11702
11703         if (tc > bp->max_tc) {
11704                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
11705                            tc, bp->max_tc);
11706                 return -EINVAL;
11707         }
11708
11709         if (netdev_get_num_tc(dev) == tc)
11710                 return 0;
11711
11712         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
11713                 sh = true;
11714
11715         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
11716                               sh, tc, bp->tx_nr_rings_xdp);
11717         if (rc)
11718                 return rc;
11719
11720         /* Needs to close the device and do hw resource re-allocations */
11721         if (netif_running(bp->dev))
11722                 bnxt_close_nic(bp, true, false);
11723
11724         if (tc) {
11725                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
11726                 netdev_set_num_tc(dev, tc);
11727         } else {
11728                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11729                 netdev_reset_tc(dev);
11730         }
11731         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
11732         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
11733                                bp->tx_nr_rings + bp->rx_nr_rings;
11734
11735         if (netif_running(bp->dev))
11736                 return bnxt_open_nic(bp, true, false);
11737
11738         return 0;
11739 }
11740
11741 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
11742                                   void *cb_priv)
11743 {
11744         struct bnxt *bp = cb_priv;
11745
11746         if (!bnxt_tc_flower_enabled(bp) ||
11747             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
11748                 return -EOPNOTSUPP;
11749
11750         switch (type) {
11751         case TC_SETUP_CLSFLOWER:
11752                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
11753         default:
11754                 return -EOPNOTSUPP;
11755         }
11756 }
11757
11758 LIST_HEAD(bnxt_block_cb_list);
11759
11760 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
11761                          void *type_data)
11762 {
11763         struct bnxt *bp = netdev_priv(dev);
11764
11765         switch (type) {
11766         case TC_SETUP_BLOCK:
11767                 return flow_block_cb_setup_simple(type_data,
11768                                                   &bnxt_block_cb_list,
11769                                                   bnxt_setup_tc_block_cb,
11770                                                   bp, bp, true);
11771         case TC_SETUP_QDISC_MQPRIO: {
11772                 struct tc_mqprio_qopt *mqprio = type_data;
11773
11774                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
11775
11776                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
11777         }
11778         default:
11779                 return -EOPNOTSUPP;
11780         }
11781 }
11782
11783 #ifdef CONFIG_RFS_ACCEL
11784 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
11785                             struct bnxt_ntuple_filter *f2)
11786 {
11787         struct flow_keys *keys1 = &f1->fkeys;
11788         struct flow_keys *keys2 = &f2->fkeys;
11789
11790         if (keys1->basic.n_proto != keys2->basic.n_proto ||
11791             keys1->basic.ip_proto != keys2->basic.ip_proto)
11792                 return false;
11793
11794         if (keys1->basic.n_proto == htons(ETH_P_IP)) {
11795                 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
11796                     keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
11797                         return false;
11798         } else {
11799                 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
11800                            sizeof(keys1->addrs.v6addrs.src)) ||
11801                     memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
11802                            sizeof(keys1->addrs.v6addrs.dst)))
11803                         return false;
11804         }
11805
11806         if (keys1->ports.ports == keys2->ports.ports &&
11807             keys1->control.flags == keys2->control.flags &&
11808             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
11809             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
11810                 return true;
11811
11812         return false;
11813 }
11814
11815 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
11816                               u16 rxq_index, u32 flow_id)
11817 {
11818         struct bnxt *bp = netdev_priv(dev);
11819         struct bnxt_ntuple_filter *fltr, *new_fltr;
11820         struct flow_keys *fkeys;
11821         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
11822         int rc = 0, idx, bit_id, l2_idx = 0;
11823         struct hlist_head *head;
11824         u32 flags;
11825
11826         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
11827                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11828                 int off = 0, j;
11829
11830                 netif_addr_lock_bh(dev);
11831                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
11832                         if (ether_addr_equal(eth->h_dest,
11833                                              vnic->uc_list + off)) {
11834                                 l2_idx = j + 1;
11835                                 break;
11836                         }
11837                 }
11838                 netif_addr_unlock_bh(dev);
11839                 if (!l2_idx)
11840                         return -EINVAL;
11841         }
11842         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
11843         if (!new_fltr)
11844                 return -ENOMEM;
11845
11846         fkeys = &new_fltr->fkeys;
11847         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
11848                 rc = -EPROTONOSUPPORT;
11849                 goto err_free;
11850         }
11851
11852         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
11853              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
11854             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
11855              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
11856                 rc = -EPROTONOSUPPORT;
11857                 goto err_free;
11858         }
11859         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
11860             bp->hwrm_spec_code < 0x10601) {
11861                 rc = -EPROTONOSUPPORT;
11862                 goto err_free;
11863         }
11864         flags = fkeys->control.flags;
11865         if (((flags & FLOW_DIS_ENCAPSULATION) &&
11866              bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
11867                 rc = -EPROTONOSUPPORT;
11868                 goto err_free;
11869         }
11870
11871         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
11872         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
11873
11874         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
11875         head = &bp->ntp_fltr_hash_tbl[idx];
11876         rcu_read_lock();
11877         hlist_for_each_entry_rcu(fltr, head, hash) {
11878                 if (bnxt_fltr_match(fltr, new_fltr)) {
11879                         rcu_read_unlock();
11880                         rc = 0;
11881                         goto err_free;
11882                 }
11883         }
11884         rcu_read_unlock();
11885
11886         spin_lock_bh(&bp->ntp_fltr_lock);
11887         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
11888                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
11889         if (bit_id < 0) {
11890                 spin_unlock_bh(&bp->ntp_fltr_lock);
11891                 rc = -ENOMEM;
11892                 goto err_free;
11893         }
11894
11895         new_fltr->sw_id = (u16)bit_id;
11896         new_fltr->flow_id = flow_id;
11897         new_fltr->l2_fltr_idx = l2_idx;
11898         new_fltr->rxq = rxq_index;
11899         hlist_add_head_rcu(&new_fltr->hash, head);
11900         bp->ntp_fltr_count++;
11901         spin_unlock_bh(&bp->ntp_fltr_lock);
11902
11903         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
11904         bnxt_queue_sp_work(bp);
11905
11906         return new_fltr->sw_id;
11907
11908 err_free:
11909         kfree(new_fltr);
11910         return rc;
11911 }
11912
11913 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11914 {
11915         int i;
11916
11917         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
11918                 struct hlist_head *head;
11919                 struct hlist_node *tmp;
11920                 struct bnxt_ntuple_filter *fltr;
11921                 int rc;
11922
11923                 head = &bp->ntp_fltr_hash_tbl[i];
11924                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
11925                         bool del = false;
11926
11927                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
11928                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
11929                                                         fltr->flow_id,
11930                                                         fltr->sw_id)) {
11931                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
11932                                                                          fltr);
11933                                         del = true;
11934                                 }
11935                         } else {
11936                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
11937                                                                        fltr);
11938                                 if (rc)
11939                                         del = true;
11940                                 else
11941                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
11942                         }
11943
11944                         if (del) {
11945                                 spin_lock_bh(&bp->ntp_fltr_lock);
11946                                 hlist_del_rcu(&fltr->hash);
11947                                 bp->ntp_fltr_count--;
11948                                 spin_unlock_bh(&bp->ntp_fltr_lock);
11949                                 synchronize_rcu();
11950                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
11951                                 kfree(fltr);
11952                         }
11953                 }
11954         }
11955         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
11956                 netdev_info(bp->dev, "Receive PF driver unload event!\n");
11957 }
11958
11959 #else
11960
11961 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11962 {
11963 }
11964
11965 #endif /* CONFIG_RFS_ACCEL */
11966
11967 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table)
11968 {
11969         struct bnxt *bp = netdev_priv(netdev);
11970         struct udp_tunnel_info ti;
11971         unsigned int cmd;
11972
11973         udp_tunnel_nic_get_port(netdev, table, 0, &ti);
11974         if (ti.type == UDP_TUNNEL_TYPE_VXLAN)
11975                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
11976         else
11977                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
11978
11979         if (ti.port)
11980                 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd);
11981
11982         return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
11983 }
11984
11985 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
11986         .sync_table     = bnxt_udp_tunnel_sync,
11987         .flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
11988                           UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
11989         .tables         = {
11990                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
11991                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
11992         },
11993 };
11994
11995 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
11996                                struct net_device *dev, u32 filter_mask,
11997                                int nlflags)
11998 {
11999         struct bnxt *bp = netdev_priv(dev);
12000
12001         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
12002                                        nlflags, filter_mask, NULL);
12003 }
12004
12005 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
12006                                u16 flags, struct netlink_ext_ack *extack)
12007 {
12008         struct bnxt *bp = netdev_priv(dev);
12009         struct nlattr *attr, *br_spec;
12010         int rem, rc = 0;
12011
12012         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
12013                 return -EOPNOTSUPP;
12014
12015         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
12016         if (!br_spec)
12017                 return -EINVAL;
12018
12019         nla_for_each_nested(attr, br_spec, rem) {
12020                 u16 mode;
12021
12022                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
12023                         continue;
12024
12025                 if (nla_len(attr) < sizeof(mode))
12026                         return -EINVAL;
12027
12028                 mode = nla_get_u16(attr);
12029                 if (mode == bp->br_mode)
12030                         break;
12031
12032                 rc = bnxt_hwrm_set_br_mode(bp, mode);
12033                 if (!rc)
12034                         bp->br_mode = mode;
12035                 break;
12036         }
12037         return rc;
12038 }
12039
12040 int bnxt_get_port_parent_id(struct net_device *dev,
12041                             struct netdev_phys_item_id *ppid)
12042 {
12043         struct bnxt *bp = netdev_priv(dev);
12044
12045         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
12046                 return -EOPNOTSUPP;
12047
12048         /* The PF and it's VF-reps only support the switchdev framework */
12049         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
12050                 return -EOPNOTSUPP;
12051
12052         ppid->id_len = sizeof(bp->dsn);
12053         memcpy(ppid->id, bp->dsn, ppid->id_len);
12054
12055         return 0;
12056 }
12057
12058 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
12059 {
12060         struct bnxt *bp = netdev_priv(dev);
12061
12062         return &bp->dl_port;
12063 }
12064
12065 static const struct net_device_ops bnxt_netdev_ops = {
12066         .ndo_open               = bnxt_open,
12067         .ndo_start_xmit         = bnxt_start_xmit,
12068         .ndo_stop               = bnxt_close,
12069         .ndo_get_stats64        = bnxt_get_stats64,
12070         .ndo_set_rx_mode        = bnxt_set_rx_mode,
12071         .ndo_do_ioctl           = bnxt_ioctl,
12072         .ndo_validate_addr      = eth_validate_addr,
12073         .ndo_set_mac_address    = bnxt_change_mac_addr,
12074         .ndo_change_mtu         = bnxt_change_mtu,
12075         .ndo_fix_features       = bnxt_fix_features,
12076         .ndo_set_features       = bnxt_set_features,
12077         .ndo_tx_timeout         = bnxt_tx_timeout,
12078 #ifdef CONFIG_BNXT_SRIOV
12079         .ndo_get_vf_config      = bnxt_get_vf_config,
12080         .ndo_set_vf_mac         = bnxt_set_vf_mac,
12081         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
12082         .ndo_set_vf_rate        = bnxt_set_vf_bw,
12083         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
12084         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
12085         .ndo_set_vf_trust       = bnxt_set_vf_trust,
12086 #endif
12087         .ndo_setup_tc           = bnxt_setup_tc,
12088 #ifdef CONFIG_RFS_ACCEL
12089         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
12090 #endif
12091         .ndo_udp_tunnel_add     = udp_tunnel_nic_add_port,
12092         .ndo_udp_tunnel_del     = udp_tunnel_nic_del_port,
12093         .ndo_bpf                = bnxt_xdp,
12094         .ndo_xdp_xmit           = bnxt_xdp_xmit,
12095         .ndo_bridge_getlink     = bnxt_bridge_getlink,
12096         .ndo_bridge_setlink     = bnxt_bridge_setlink,
12097         .ndo_get_devlink_port   = bnxt_get_devlink_port,
12098 };
12099
12100 static void bnxt_remove_one(struct pci_dev *pdev)
12101 {
12102         struct net_device *dev = pci_get_drvdata(pdev);
12103         struct bnxt *bp = netdev_priv(dev);
12104
12105         if (BNXT_PF(bp))
12106                 bnxt_sriov_disable(bp);
12107
12108         if (BNXT_PF(bp))
12109                 devlink_port_type_clear(&bp->dl_port);
12110         pci_disable_pcie_error_reporting(pdev);
12111         unregister_netdev(dev);
12112         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12113         /* Flush any pending tasks */
12114         cancel_work_sync(&bp->sp_task);
12115         cancel_delayed_work_sync(&bp->fw_reset_task);
12116         bp->sp_event = 0;
12117
12118         bnxt_dl_fw_reporters_destroy(bp, true);
12119         bnxt_dl_unregister(bp);
12120         bnxt_shutdown_tc(bp);
12121
12122         bnxt_clear_int_mode(bp);
12123         bnxt_hwrm_func_drv_unrgtr(bp);
12124         bnxt_free_hwrm_resources(bp);
12125         bnxt_free_hwrm_short_cmd_req(bp);
12126         bnxt_ethtool_free(bp);
12127         bnxt_dcb_free(bp);
12128         kfree(bp->edev);
12129         bp->edev = NULL;
12130         kfree(bp->fw_health);
12131         bp->fw_health = NULL;
12132         bnxt_cleanup_pci(bp);
12133         bnxt_free_ctx_mem(bp);
12134         kfree(bp->ctx);
12135         bp->ctx = NULL;
12136         kfree(bp->rss_indir_tbl);
12137         bp->rss_indir_tbl = NULL;
12138         bnxt_free_port_stats(bp);
12139         free_netdev(dev);
12140 }
12141
12142 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
12143 {
12144         int rc = 0;
12145         struct bnxt_link_info *link_info = &bp->link_info;
12146
12147         rc = bnxt_hwrm_phy_qcaps(bp);
12148         if (rc) {
12149                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
12150                            rc);
12151                 return rc;
12152         }
12153         if (!fw_dflt)
12154                 return 0;
12155
12156         rc = bnxt_update_link(bp, false);
12157         if (rc) {
12158                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
12159                            rc);
12160                 return rc;
12161         }
12162
12163         /* Older firmware does not have supported_auto_speeds, so assume
12164          * that all supported speeds can be autonegotiated.
12165          */
12166         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
12167                 link_info->support_auto_speeds = link_info->support_speeds;
12168
12169         bnxt_init_ethtool_link_settings(bp);
12170         return 0;
12171 }
12172
12173 static int bnxt_get_max_irq(struct pci_dev *pdev)
12174 {
12175         u16 ctrl;
12176
12177         if (!pdev->msix_cap)
12178                 return 1;
12179
12180         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
12181         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
12182 }
12183
12184 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
12185                                 int *max_cp)
12186 {
12187         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
12188         int max_ring_grps = 0, max_irq;
12189
12190         *max_tx = hw_resc->max_tx_rings;
12191         *max_rx = hw_resc->max_rx_rings;
12192         *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
12193         max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
12194                         bnxt_get_ulp_msix_num(bp),
12195                         hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
12196         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
12197                 *max_cp = min_t(int, *max_cp, max_irq);
12198         max_ring_grps = hw_resc->max_hw_ring_grps;
12199         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
12200                 *max_cp -= 1;
12201                 *max_rx -= 2;
12202         }
12203         if (bp->flags & BNXT_FLAG_AGG_RINGS)
12204                 *max_rx >>= 1;
12205         if (bp->flags & BNXT_FLAG_CHIP_P5) {
12206                 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
12207                 /* On P5 chips, max_cp output param should be available NQs */
12208                 *max_cp = max_irq;
12209         }
12210         *max_rx = min_t(int, *max_rx, max_ring_grps);
12211 }
12212
12213 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
12214 {
12215         int rx, tx, cp;
12216
12217         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
12218         *max_rx = rx;
12219         *max_tx = tx;
12220         if (!rx || !tx || !cp)
12221                 return -ENOMEM;
12222
12223         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
12224 }
12225
12226 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
12227                                bool shared)
12228 {
12229         int rc;
12230
12231         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
12232         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
12233                 /* Not enough rings, try disabling agg rings. */
12234                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
12235                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
12236                 if (rc) {
12237                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
12238                         bp->flags |= BNXT_FLAG_AGG_RINGS;
12239                         return rc;
12240                 }
12241                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
12242                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12243                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12244                 bnxt_set_ring_params(bp);
12245         }
12246
12247         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
12248                 int max_cp, max_stat, max_irq;
12249
12250                 /* Reserve minimum resources for RoCE */
12251                 max_cp = bnxt_get_max_func_cp_rings(bp);
12252                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
12253                 max_irq = bnxt_get_max_func_irqs(bp);
12254                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
12255                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
12256                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
12257                         return 0;
12258
12259                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
12260                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
12261                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
12262                 max_cp = min_t(int, max_cp, max_irq);
12263                 max_cp = min_t(int, max_cp, max_stat);
12264                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
12265                 if (rc)
12266                         rc = 0;
12267         }
12268         return rc;
12269 }
12270
12271 /* In initial default shared ring setting, each shared ring must have a
12272  * RX/TX ring pair.
12273  */
12274 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
12275 {
12276         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
12277         bp->rx_nr_rings = bp->cp_nr_rings;
12278         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
12279         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12280 }
12281
12282 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
12283 {
12284         int dflt_rings, max_rx_rings, max_tx_rings, rc;
12285
12286         if (!bnxt_can_reserve_rings(bp))
12287                 return 0;
12288
12289         if (sh)
12290                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
12291         dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
12292         /* Reduce default rings on multi-port cards so that total default
12293          * rings do not exceed CPU count.
12294          */
12295         if (bp->port_count > 1) {
12296                 int max_rings =
12297                         max_t(int, num_online_cpus() / bp->port_count, 1);
12298
12299                 dflt_rings = min_t(int, dflt_rings, max_rings);
12300         }
12301         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
12302         if (rc)
12303                 return rc;
12304         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
12305         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
12306         if (sh)
12307                 bnxt_trim_dflt_sh_rings(bp);
12308         else
12309                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
12310         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12311
12312         rc = __bnxt_reserve_rings(bp);
12313         if (rc)
12314                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
12315         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12316         if (sh)
12317                 bnxt_trim_dflt_sh_rings(bp);
12318
12319         /* Rings may have been trimmed, re-reserve the trimmed rings. */
12320         if (bnxt_need_reserve_rings(bp)) {
12321                 rc = __bnxt_reserve_rings(bp);
12322                 if (rc)
12323                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
12324                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12325         }
12326         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
12327                 bp->rx_nr_rings++;
12328                 bp->cp_nr_rings++;
12329         }
12330         if (rc) {
12331                 bp->tx_nr_rings = 0;
12332                 bp->rx_nr_rings = 0;
12333         }
12334         return rc;
12335 }
12336
12337 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
12338 {
12339         int rc;
12340
12341         if (bp->tx_nr_rings)
12342                 return 0;
12343
12344         bnxt_ulp_irq_stop(bp);
12345         bnxt_clear_int_mode(bp);
12346         rc = bnxt_set_dflt_rings(bp, true);
12347         if (rc) {
12348                 netdev_err(bp->dev, "Not enough rings available.\n");
12349                 goto init_dflt_ring_err;
12350         }
12351         rc = bnxt_init_int_mode(bp);
12352         if (rc)
12353                 goto init_dflt_ring_err;
12354
12355         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12356         if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
12357                 bp->flags |= BNXT_FLAG_RFS;
12358                 bp->dev->features |= NETIF_F_NTUPLE;
12359         }
12360 init_dflt_ring_err:
12361         bnxt_ulp_irq_restart(bp, rc);
12362         return rc;
12363 }
12364
12365 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
12366 {
12367         int rc;
12368
12369         ASSERT_RTNL();
12370         bnxt_hwrm_func_qcaps(bp);
12371
12372         if (netif_running(bp->dev))
12373                 __bnxt_close_nic(bp, true, false);
12374
12375         bnxt_ulp_irq_stop(bp);
12376         bnxt_clear_int_mode(bp);
12377         rc = bnxt_init_int_mode(bp);
12378         bnxt_ulp_irq_restart(bp, rc);
12379
12380         if (netif_running(bp->dev)) {
12381                 if (rc)
12382                         dev_close(bp->dev);
12383                 else
12384                         rc = bnxt_open_nic(bp, true, false);
12385         }
12386
12387         return rc;
12388 }
12389
12390 static int bnxt_init_mac_addr(struct bnxt *bp)
12391 {
12392         int rc = 0;
12393
12394         if (BNXT_PF(bp)) {
12395                 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
12396         } else {
12397 #ifdef CONFIG_BNXT_SRIOV
12398                 struct bnxt_vf_info *vf = &bp->vf;
12399                 bool strict_approval = true;
12400
12401                 if (is_valid_ether_addr(vf->mac_addr)) {
12402                         /* overwrite netdev dev_addr with admin VF MAC */
12403                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
12404                         /* Older PF driver or firmware may not approve this
12405                          * correctly.
12406                          */
12407                         strict_approval = false;
12408                 } else {
12409                         eth_hw_addr_random(bp->dev);
12410                 }
12411                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
12412 #endif
12413         }
12414         return rc;
12415 }
12416
12417 #define BNXT_VPD_LEN    512
12418 static void bnxt_vpd_read_info(struct bnxt *bp)
12419 {
12420         struct pci_dev *pdev = bp->pdev;
12421         int i, len, pos, ro_size, size;
12422         ssize_t vpd_size;
12423         u8 *vpd_data;
12424
12425         vpd_data = kmalloc(BNXT_VPD_LEN, GFP_KERNEL);
12426         if (!vpd_data)
12427                 return;
12428
12429         vpd_size = pci_read_vpd(pdev, 0, BNXT_VPD_LEN, vpd_data);
12430         if (vpd_size <= 0) {
12431                 netdev_err(bp->dev, "Unable to read VPD\n");
12432                 goto exit;
12433         }
12434
12435         i = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA);
12436         if (i < 0) {
12437                 netdev_err(bp->dev, "VPD READ-Only not found\n");
12438                 goto exit;
12439         }
12440
12441         ro_size = pci_vpd_lrdt_size(&vpd_data[i]);
12442         i += PCI_VPD_LRDT_TAG_SIZE;
12443         if (i + ro_size > vpd_size)
12444                 goto exit;
12445
12446         pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12447                                         PCI_VPD_RO_KEYWORD_PARTNO);
12448         if (pos < 0)
12449                 goto read_sn;
12450
12451         len = pci_vpd_info_field_size(&vpd_data[pos]);
12452         pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12453         if (len + pos > vpd_size)
12454                 goto read_sn;
12455
12456         size = min(len, BNXT_VPD_FLD_LEN - 1);
12457         memcpy(bp->board_partno, &vpd_data[pos], size);
12458
12459 read_sn:
12460         pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12461                                         PCI_VPD_RO_KEYWORD_SERIALNO);
12462         if (pos < 0)
12463                 goto exit;
12464
12465         len = pci_vpd_info_field_size(&vpd_data[pos]);
12466         pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12467         if (len + pos > vpd_size)
12468                 goto exit;
12469
12470         size = min(len, BNXT_VPD_FLD_LEN - 1);
12471         memcpy(bp->board_serialno, &vpd_data[pos], size);
12472 exit:
12473         kfree(vpd_data);
12474 }
12475
12476 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
12477 {
12478         struct pci_dev *pdev = bp->pdev;
12479         u64 qword;
12480
12481         qword = pci_get_dsn(pdev);
12482         if (!qword) {
12483                 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
12484                 return -EOPNOTSUPP;
12485         }
12486
12487         put_unaligned_le64(qword, dsn);
12488
12489         bp->flags |= BNXT_FLAG_DSN_VALID;
12490         return 0;
12491 }
12492
12493 static int bnxt_map_db_bar(struct bnxt *bp)
12494 {
12495         if (!bp->db_size)
12496                 return -ENODEV;
12497         bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
12498         if (!bp->bar1)
12499                 return -ENOMEM;
12500         return 0;
12501 }
12502
12503 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
12504 {
12505         struct net_device *dev;
12506         struct bnxt *bp;
12507         int rc, max_irqs;
12508
12509         if (pci_is_bridge(pdev))
12510                 return -ENODEV;
12511
12512         /* Clear any pending DMA transactions from crash kernel
12513          * while loading driver in capture kernel.
12514          */
12515         if (is_kdump_kernel()) {
12516                 pci_clear_master(pdev);
12517                 pcie_flr(pdev);
12518         }
12519
12520         max_irqs = bnxt_get_max_irq(pdev);
12521         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
12522         if (!dev)
12523                 return -ENOMEM;
12524
12525         bp = netdev_priv(dev);
12526         bp->msg_enable = BNXT_DEF_MSG_ENABLE;
12527         bnxt_set_max_func_irqs(bp, max_irqs);
12528
12529         if (bnxt_vf_pciid(ent->driver_data))
12530                 bp->flags |= BNXT_FLAG_VF;
12531
12532         if (pdev->msix_cap)
12533                 bp->flags |= BNXT_FLAG_MSIX_CAP;
12534
12535         rc = bnxt_init_board(pdev, dev);
12536         if (rc < 0)
12537                 goto init_err_free;
12538
12539         dev->netdev_ops = &bnxt_netdev_ops;
12540         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
12541         dev->ethtool_ops = &bnxt_ethtool_ops;
12542         pci_set_drvdata(pdev, dev);
12543
12544         if (BNXT_PF(bp))
12545                 bnxt_vpd_read_info(bp);
12546
12547         rc = bnxt_alloc_hwrm_resources(bp);
12548         if (rc)
12549                 goto init_err_pci_clean;
12550
12551         mutex_init(&bp->hwrm_cmd_lock);
12552         mutex_init(&bp->link_lock);
12553
12554         rc = bnxt_fw_init_one_p1(bp);
12555         if (rc)
12556                 goto init_err_pci_clean;
12557
12558         if (BNXT_CHIP_P5(bp)) {
12559                 bp->flags |= BNXT_FLAG_CHIP_P5;
12560                 if (BNXT_CHIP_SR2(bp))
12561                         bp->flags |= BNXT_FLAG_CHIP_SR2;
12562         }
12563
12564         rc = bnxt_alloc_rss_indir_tbl(bp);
12565         if (rc)
12566                 goto init_err_pci_clean;
12567
12568         rc = bnxt_fw_init_one_p2(bp);
12569         if (rc)
12570                 goto init_err_pci_clean;
12571
12572         rc = bnxt_map_db_bar(bp);
12573         if (rc) {
12574                 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
12575                         rc);
12576                 goto init_err_pci_clean;
12577         }
12578
12579         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12580                            NETIF_F_TSO | NETIF_F_TSO6 |
12581                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12582                            NETIF_F_GSO_IPXIP4 |
12583                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12584                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
12585                            NETIF_F_RXCSUM | NETIF_F_GRO;
12586
12587         if (BNXT_SUPPORTS_TPA(bp))
12588                 dev->hw_features |= NETIF_F_LRO;
12589
12590         dev->hw_enc_features =
12591                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12592                         NETIF_F_TSO | NETIF_F_TSO6 |
12593                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12594                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12595                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
12596         dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
12597
12598         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
12599                                     NETIF_F_GSO_GRE_CSUM;
12600         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
12601         if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
12602                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
12603         if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
12604                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
12605         if (BNXT_SUPPORTS_TPA(bp))
12606                 dev->hw_features |= NETIF_F_GRO_HW;
12607         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
12608         if (dev->features & NETIF_F_GRO_HW)
12609                 dev->features &= ~NETIF_F_LRO;
12610         dev->priv_flags |= IFF_UNICAST_FLT;
12611
12612 #ifdef CONFIG_BNXT_SRIOV
12613         init_waitqueue_head(&bp->sriov_cfg_wait);
12614         mutex_init(&bp->sriov_lock);
12615 #endif
12616         if (BNXT_SUPPORTS_TPA(bp)) {
12617                 bp->gro_func = bnxt_gro_func_5730x;
12618                 if (BNXT_CHIP_P4(bp))
12619                         bp->gro_func = bnxt_gro_func_5731x;
12620                 else if (BNXT_CHIP_P5(bp))
12621                         bp->gro_func = bnxt_gro_func_5750x;
12622         }
12623         if (!BNXT_CHIP_P4_PLUS(bp))
12624                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
12625
12626         bp->ulp_probe = bnxt_ulp_probe;
12627
12628         rc = bnxt_init_mac_addr(bp);
12629         if (rc) {
12630                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
12631                 rc = -EADDRNOTAVAIL;
12632                 goto init_err_pci_clean;
12633         }
12634
12635         if (BNXT_PF(bp)) {
12636                 /* Read the adapter's DSN to use as the eswitch switch_id */
12637                 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
12638         }
12639
12640         /* MTU range: 60 - FW defined max */
12641         dev->min_mtu = ETH_ZLEN;
12642         dev->max_mtu = bp->max_mtu;
12643
12644         rc = bnxt_probe_phy(bp, true);
12645         if (rc)
12646                 goto init_err_pci_clean;
12647
12648         bnxt_set_rx_skb_mode(bp, false);
12649         bnxt_set_tpa_flags(bp);
12650         bnxt_set_ring_params(bp);
12651         rc = bnxt_set_dflt_rings(bp, true);
12652         if (rc) {
12653                 netdev_err(bp->dev, "Not enough rings available.\n");
12654                 rc = -ENOMEM;
12655                 goto init_err_pci_clean;
12656         }
12657
12658         bnxt_fw_init_one_p3(bp);
12659
12660         if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12661                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
12662
12663         rc = bnxt_init_int_mode(bp);
12664         if (rc)
12665                 goto init_err_pci_clean;
12666
12667         /* No TC has been set yet and rings may have been trimmed due to
12668          * limited MSIX, so we re-initialize the TX rings per TC.
12669          */
12670         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12671
12672         if (BNXT_PF(bp)) {
12673                 if (!bnxt_pf_wq) {
12674                         bnxt_pf_wq =
12675                                 create_singlethread_workqueue("bnxt_pf_wq");
12676                         if (!bnxt_pf_wq) {
12677                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
12678                                 rc = -ENOMEM;
12679                                 goto init_err_pci_clean;
12680                         }
12681                 }
12682                 rc = bnxt_init_tc(bp);
12683                 if (rc)
12684                         netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
12685                                    rc);
12686         }
12687
12688         bnxt_dl_register(bp);
12689
12690         rc = register_netdev(dev);
12691         if (rc)
12692                 goto init_err_cleanup;
12693
12694         if (BNXT_PF(bp))
12695                 devlink_port_type_eth_set(&bp->dl_port, bp->dev);
12696         bnxt_dl_fw_reporters_create(bp);
12697
12698         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
12699                     board_info[ent->driver_data].name,
12700                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
12701         pcie_print_link_status(pdev);
12702
12703         pci_save_state(pdev);
12704         return 0;
12705
12706 init_err_cleanup:
12707         bnxt_dl_unregister(bp);
12708         bnxt_shutdown_tc(bp);
12709         bnxt_clear_int_mode(bp);
12710
12711 init_err_pci_clean:
12712         bnxt_hwrm_func_drv_unrgtr(bp);
12713         bnxt_free_hwrm_short_cmd_req(bp);
12714         bnxt_free_hwrm_resources(bp);
12715         kfree(bp->fw_health);
12716         bp->fw_health = NULL;
12717         bnxt_cleanup_pci(bp);
12718         bnxt_free_ctx_mem(bp);
12719         kfree(bp->ctx);
12720         bp->ctx = NULL;
12721         kfree(bp->rss_indir_tbl);
12722         bp->rss_indir_tbl = NULL;
12723
12724 init_err_free:
12725         free_netdev(dev);
12726         return rc;
12727 }
12728
12729 static void bnxt_shutdown(struct pci_dev *pdev)
12730 {
12731         struct net_device *dev = pci_get_drvdata(pdev);
12732         struct bnxt *bp;
12733
12734         if (!dev)
12735                 return;
12736
12737         rtnl_lock();
12738         bp = netdev_priv(dev);
12739         if (!bp)
12740                 goto shutdown_exit;
12741
12742         if (netif_running(dev))
12743                 dev_close(dev);
12744
12745         bnxt_ulp_shutdown(bp);
12746         bnxt_clear_int_mode(bp);
12747         pci_disable_device(pdev);
12748
12749         if (system_state == SYSTEM_POWER_OFF) {
12750                 pci_wake_from_d3(pdev, bp->wol);
12751                 pci_set_power_state(pdev, PCI_D3hot);
12752         }
12753
12754 shutdown_exit:
12755         rtnl_unlock();
12756 }
12757
12758 #ifdef CONFIG_PM_SLEEP
12759 static int bnxt_suspend(struct device *device)
12760 {
12761         struct net_device *dev = dev_get_drvdata(device);
12762         struct bnxt *bp = netdev_priv(dev);
12763         int rc = 0;
12764
12765         rtnl_lock();
12766         bnxt_ulp_stop(bp);
12767         if (netif_running(dev)) {
12768                 netif_device_detach(dev);
12769                 rc = bnxt_close(dev);
12770         }
12771         bnxt_hwrm_func_drv_unrgtr(bp);
12772         pci_disable_device(bp->pdev);
12773         bnxt_free_ctx_mem(bp);
12774         kfree(bp->ctx);
12775         bp->ctx = NULL;
12776         rtnl_unlock();
12777         return rc;
12778 }
12779
12780 static int bnxt_resume(struct device *device)
12781 {
12782         struct net_device *dev = dev_get_drvdata(device);
12783         struct bnxt *bp = netdev_priv(dev);
12784         int rc = 0;
12785
12786         rtnl_lock();
12787         rc = pci_enable_device(bp->pdev);
12788         if (rc) {
12789                 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
12790                            rc);
12791                 goto resume_exit;
12792         }
12793         pci_set_master(bp->pdev);
12794         if (bnxt_hwrm_ver_get(bp)) {
12795                 rc = -ENODEV;
12796                 goto resume_exit;
12797         }
12798         rc = bnxt_hwrm_func_reset(bp);
12799         if (rc) {
12800                 rc = -EBUSY;
12801                 goto resume_exit;
12802         }
12803
12804         rc = bnxt_hwrm_func_qcaps(bp);
12805         if (rc)
12806                 goto resume_exit;
12807
12808         if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
12809                 rc = -ENODEV;
12810                 goto resume_exit;
12811         }
12812
12813         bnxt_get_wol_settings(bp);
12814         if (netif_running(dev)) {
12815                 rc = bnxt_open(dev);
12816                 if (!rc)
12817                         netif_device_attach(dev);
12818         }
12819
12820 resume_exit:
12821         bnxt_ulp_start(bp, rc);
12822         if (!rc)
12823                 bnxt_reenable_sriov(bp);
12824         rtnl_unlock();
12825         return rc;
12826 }
12827
12828 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
12829 #define BNXT_PM_OPS (&bnxt_pm_ops)
12830
12831 #else
12832
12833 #define BNXT_PM_OPS NULL
12834
12835 #endif /* CONFIG_PM_SLEEP */
12836
12837 /**
12838  * bnxt_io_error_detected - called when PCI error is detected
12839  * @pdev: Pointer to PCI device
12840  * @state: The current pci connection state
12841  *
12842  * This function is called after a PCI bus error affecting
12843  * this device has been detected.
12844  */
12845 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
12846                                                pci_channel_state_t state)
12847 {
12848         struct net_device *netdev = pci_get_drvdata(pdev);
12849         struct bnxt *bp = netdev_priv(netdev);
12850
12851         netdev_info(netdev, "PCI I/O error detected\n");
12852
12853         rtnl_lock();
12854         netif_device_detach(netdev);
12855
12856         bnxt_ulp_stop(bp);
12857
12858         if (state == pci_channel_io_perm_failure) {
12859                 rtnl_unlock();
12860                 return PCI_ERS_RESULT_DISCONNECT;
12861         }
12862
12863         if (state == pci_channel_io_frozen)
12864                 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
12865
12866         if (netif_running(netdev))
12867                 bnxt_close(netdev);
12868
12869         pci_disable_device(pdev);
12870         bnxt_free_ctx_mem(bp);
12871         kfree(bp->ctx);
12872         bp->ctx = NULL;
12873         rtnl_unlock();
12874
12875         /* Request a slot slot reset. */
12876         return PCI_ERS_RESULT_NEED_RESET;
12877 }
12878
12879 /**
12880  * bnxt_io_slot_reset - called after the pci bus has been reset.
12881  * @pdev: Pointer to PCI device
12882  *
12883  * Restart the card from scratch, as if from a cold-boot.
12884  * At this point, the card has exprienced a hard reset,
12885  * followed by fixups by BIOS, and has its config space
12886  * set up identically to what it was at cold boot.
12887  */
12888 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
12889 {
12890         struct net_device *netdev = pci_get_drvdata(pdev);
12891         struct bnxt *bp = netdev_priv(netdev);
12892         int err = 0, off;
12893         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
12894
12895         netdev_info(bp->dev, "PCI Slot Reset\n");
12896
12897         rtnl_lock();
12898
12899         if (pci_enable_device(pdev)) {
12900                 dev_err(&pdev->dev,
12901                         "Cannot re-enable PCI device after reset.\n");
12902         } else {
12903                 pci_set_master(pdev);
12904                 /* Upon fatal error, our device internal logic that latches to
12905                  * BAR value is getting reset and will restore only upon
12906                  * rewritting the BARs.
12907                  *
12908                  * As pci_restore_state() does not re-write the BARs if the
12909                  * value is same as saved value earlier, driver needs to
12910                  * write the BARs to 0 to force restore, in case of fatal error.
12911                  */
12912                 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
12913                                        &bp->state)) {
12914                         for (off = PCI_BASE_ADDRESS_0;
12915                              off <= PCI_BASE_ADDRESS_5; off += 4)
12916                                 pci_write_config_dword(bp->pdev, off, 0);
12917                 }
12918                 pci_restore_state(pdev);
12919                 pci_save_state(pdev);
12920
12921                 err = bnxt_hwrm_func_reset(bp);
12922                 if (!err) {
12923                         err = bnxt_hwrm_func_qcaps(bp);
12924                         if (!err && netif_running(netdev))
12925                                 err = bnxt_open(netdev);
12926                 }
12927                 bnxt_ulp_start(bp, err);
12928                 if (!err) {
12929                         bnxt_reenable_sriov(bp);
12930                         result = PCI_ERS_RESULT_RECOVERED;
12931                 }
12932         }
12933
12934         if (result != PCI_ERS_RESULT_RECOVERED) {
12935                 if (netif_running(netdev))
12936                         dev_close(netdev);
12937                 pci_disable_device(pdev);
12938         }
12939
12940         rtnl_unlock();
12941
12942         return result;
12943 }
12944
12945 /**
12946  * bnxt_io_resume - called when traffic can start flowing again.
12947  * @pdev: Pointer to PCI device
12948  *
12949  * This callback is called when the error recovery driver tells
12950  * us that its OK to resume normal operation.
12951  */
12952 static void bnxt_io_resume(struct pci_dev *pdev)
12953 {
12954         struct net_device *netdev = pci_get_drvdata(pdev);
12955
12956         rtnl_lock();
12957
12958         netif_device_attach(netdev);
12959
12960         rtnl_unlock();
12961 }
12962
12963 static const struct pci_error_handlers bnxt_err_handler = {
12964         .error_detected = bnxt_io_error_detected,
12965         .slot_reset     = bnxt_io_slot_reset,
12966         .resume         = bnxt_io_resume
12967 };
12968
12969 static struct pci_driver bnxt_pci_driver = {
12970         .name           = DRV_MODULE_NAME,
12971         .id_table       = bnxt_pci_tbl,
12972         .probe          = bnxt_init_one,
12973         .remove         = bnxt_remove_one,
12974         .shutdown       = bnxt_shutdown,
12975         .driver.pm      = BNXT_PM_OPS,
12976         .err_handler    = &bnxt_err_handler,
12977 #if defined(CONFIG_BNXT_SRIOV)
12978         .sriov_configure = bnxt_sriov_configure,
12979 #endif
12980 };
12981
12982 static int __init bnxt_init(void)
12983 {
12984         bnxt_debug_init();
12985         return pci_register_driver(&bnxt_pci_driver);
12986 }
12987
12988 static void __exit bnxt_exit(void)
12989 {
12990         pci_unregister_driver(&bnxt_pci_driver);
12991         if (bnxt_pf_wq)
12992                 destroy_workqueue(bnxt_pf_wq);
12993         bnxt_debug_exit();
12994 }
12995
12996 module_init(bnxt_init);
12997 module_exit(bnxt_exit);