drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2019 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/mdio.h>
  35 #include <linux/if.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/if_bridge.h>
  38 #include <linux/rtc.h>
  39 #include <linux/bpf.h>
  40 #include <net/gro.h>
  41 #include <net/ip.h>
  42 #include <net/tcp.h>
  43 #include <net/udp.h>
  44 #include <net/checksum.h>
  45 #include <net/ip6_checksum.h>
  46 #include <net/udp_tunnel.h>
  47 #include <linux/workqueue.h>
  48 #include <linux/prefetch.h>
  49 #include <linux/cache.h>
  50 #include <linux/log2.h>
  51 #include <linux/bitmap.h>
  52 #include <linux/cpu_rmap.h>
  53 #include <linux/cpumask.h>
  54 #include <net/pkt_cls.h>
  55 #include <linux/hwmon.h>
  56 #include <linux/hwmon-sysfs.h>
  57 #include <net/page_pool.h>
  58 #include <linux/align.h>
  59 #include <net/netdev_queues.h>
  60
  61 #include "bnxt_hsi.h"
  62 #include "bnxt.h"
  63 #include "bnxt_hwrm.h"
  64 #include "bnxt_ulp.h"
  65 #include "bnxt_sriov.h"
  66 #include "bnxt_ethtool.h"
  67 #include "bnxt_dcb.h"
  68 #include "bnxt_xdp.h"
  69 #include "bnxt_ptp.h"
  70 #include "bnxt_vfr.h"
  71 #include "bnxt_tc.h"
  72 #include "bnxt_devlink.h"
  73 #include "bnxt_debugfs.h"
  74
  75 #define BNXT_TX_TIMEOUT         (5 * HZ)
  76 #define BNXT_DEF_MSG_ENABLE     (NETIF_MSG_DRV | NETIF_MSG_HW | \
  77                                  NETIF_MSG_TX_ERR)
  78
  79 MODULE_LICENSE("GPL");
  80 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  81
  82 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  83 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  84 #define BNXT_RX_COPY_THRESH 256
  85
  86 #define BNXT_TX_PUSH_THRESH 164
  87
  88 /* indexed by enum board_idx */
  89 static const struct {
  90         char *name;
  91 } board_info[] = {
  92         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
  93         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
  94         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
  95         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
  96         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
  97         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
  98         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
  99         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 100         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 101         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 102         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 103         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 104         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 105         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 106         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 107         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 108         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 109         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 110         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 111         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 112         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 113         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 114         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 115         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 116         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 117         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 118         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 119         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 120         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 121         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 122         [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 123         [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
 124         [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
 125         [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
 126         [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
 127         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 128         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 129         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 130         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 131         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 132         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 133         [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" },
 134         [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" },
 135         [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
 136         [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" },
 137 };
 138
 139 static const struct pci_device_id bnxt_pci_tbl[] = {
 140         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 141         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 142         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 143         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 144         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 145         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 146         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 147         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 148         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 149         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 150         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 151         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 152         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 153         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 154         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 155         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 156         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 157         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 158         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 159         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 160         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 161         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 162         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 163         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 164         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 165         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 166         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 167         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 168         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 169         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 170         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 171         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 172         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 173         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 174         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 175         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 176         { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
 177         { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
 178         { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57502_NPAR },
 179         { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
 180         { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57508_NPAR },
 181         { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57502_NPAR },
 182         { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
 183         { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57508_NPAR },
 184         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 185         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 186 #ifdef CONFIG_BNXT_SRIOV
 187         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 188         { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV },
 189         { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV },
 190         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 191         { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV },
 192         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 193         { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV },
 194         { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV },
 195         { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV },
 196         { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV },
 197         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 198         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 199         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 200         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 201         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 202         { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV },
 203         { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
 204         { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
 205         { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV },
 206         { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV },
 207         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 208 #endif
 209         { 0 }
 210 };
 211
 212 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 213
 214 static const u16 bnxt_vf_req_snif[] = {
 215         HWRM_FUNC_CFG,
 216         HWRM_FUNC_VF_CFG,
 217         HWRM_PORT_PHY_QCFG,
 218         HWRM_CFA_L2_FILTER_ALLOC,
 219 };
 220
 221 static const u16 bnxt_async_events_arr[] = {
 222         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 223         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
 224         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 225         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 226         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 227         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 228         ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
 229         ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
 230         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
 231         ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION,
 232         ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE,
 233         ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
 234         ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST,
 235         ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP,
 236         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT,
 237         ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE,
 238 };
 239
 240 static struct workqueue_struct *bnxt_pf_wq;
 241
 242 static bool bnxt_vf_pciid(enum board_idx idx)
 243 {
 244         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 245                 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV ||
 246                 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF ||
 247                 idx == NETXTREME_E_P5_VF_HV);
 248 }
 249
 250 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 251 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 252 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 253
 254 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 255                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 256
 257 #define BNXT_DB_CQ(db, idx)                                             \
 258         writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
 259
 260 #define BNXT_DB_NQ_P5(db, idx)                                          \
 261         bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx),   \
 262                     (db)->doorbell)
 263
 264 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 265         writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
 266
 267 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 268         bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx),\
 269                     (db)->doorbell)
 270
 271 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 272 {
 273         if (bp->flags & BNXT_FLAG_CHIP_P5)
 274                 BNXT_DB_NQ_P5(db, idx);
 275         else
 276                 BNXT_DB_CQ(db, idx);
 277 }
 278
 279 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 280 {
 281         if (bp->flags & BNXT_FLAG_CHIP_P5)
 282                 BNXT_DB_NQ_ARM_P5(db, idx);
 283         else
 284                 BNXT_DB_CQ_ARM(db, idx);
 285 }
 286
 287 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 288 {
 289         if (bp->flags & BNXT_FLAG_CHIP_P5)
 290                 bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL |
 291                             RING_CMP(idx), db->doorbell);
 292         else
 293                 BNXT_DB_CQ(db, idx);
 294 }
 295
 296 const u16 bnxt_lhint_arr[] = {
 297         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 298         TX_BD_FLAGS_LHINT_512_TO_1023,
 299         TX_BD_FLAGS_LHINT_1024_TO_2047,
 300         TX_BD_FLAGS_LHINT_1024_TO_2047,
 301         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 302         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 303         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 304         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 305         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 306         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 307         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 308         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 309         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 310         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 311         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 312         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 313         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 314         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 315         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 316 };
 317
 318 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 319 {
 320         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 321
 322         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 323                 return 0;
 324
 325         return md_dst->u.port_info.port_id;
 326 }
 327
 328 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
 329                              u16 prod)
 330 {
 331         bnxt_db_write(bp, &txr->tx_db, prod);
 332         txr->kick_pending = 0;
 333 }
 334
 335 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 336 {
 337         struct bnxt *bp = netdev_priv(dev);
 338         struct tx_bd *txbd;
 339         struct tx_bd_ext *txbd1;
 340         struct netdev_queue *txq;
 341         int i;
 342         dma_addr_t mapping;
 343         unsigned int length, pad = 0;
 344         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 345         u16 prod, last_frag;
 346         struct pci_dev *pdev = bp->pdev;
 347         struct bnxt_tx_ring_info *txr;
 348         struct bnxt_sw_tx_bd *tx_buf;
 349         __le32 lflags = 0;
 350
 351         i = skb_get_queue_mapping(skb);
 352         if (unlikely(i >= bp->tx_nr_rings)) {
 353                 dev_kfree_skb_any(skb);
 354                 dev_core_stats_tx_dropped_inc(dev);
 355                 return NETDEV_TX_OK;
 356         }
 357
 358         txq = netdev_get_tx_queue(dev, i);
 359         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 360         prod = txr->tx_prod;
 361
 362         free_size = bnxt_tx_avail(bp, txr);
 363         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 364                 /* We must have raced with NAPI cleanup */
 365                 if (net_ratelimit() && txr->kick_pending)
 366                         netif_warn(bp, tx_err, dev,
 367                                    "bnxt: ring busy w/ flush pending!\n");
 368                 if (!netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
 369                                         bp->tx_wake_thresh))
 370                         return NETDEV_TX_BUSY;
 371         }
 372
 373         if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
 374                 goto tx_free;
 375
 376         length = skb->len;
 377         len = skb_headlen(skb);
 378         last_frag = skb_shinfo(skb)->nr_frags;
 379
 380         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 381
 382         txbd->tx_bd_opaque = prod;
 383
 384         tx_buf = &txr->tx_buf_ring[prod];
 385         tx_buf->skb = skb;
 386         tx_buf->nr_frags = last_frag;
 387
 388         vlan_tag_flags = 0;
 389         cfa_action = bnxt_xmit_get_cfa_action(skb);
 390         if (skb_vlan_tag_present(skb)) {
 391                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 392                                  skb_vlan_tag_get(skb);
 393                 /* Currently supports 8021Q, 8021AD vlan offloads
 394                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 395                  */
 396                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 397                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 398         }
 399
 400         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
 401                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
 402
 403                 if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) &&
 404                     atomic_dec_if_positive(&ptp->tx_avail) >= 0) {
 405                         if (!bnxt_ptp_parse(skb, &ptp->tx_seqid,
 406                                             &ptp->tx_hdr_off)) {
 407                                 if (vlan_tag_flags)
 408                                         ptp->tx_hdr_off += VLAN_HLEN;
 409                                 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP);
 410                                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 411                         } else {
 412                                 atomic_inc(&bp->ptp_cfg->tx_avail);
 413                         }
 414                 }
 415         }
 416
 417         if (unlikely(skb->no_fcs))
 418                 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC);
 419
 420         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh &&
 421             !lflags) {
 422                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 423                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 424                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 425                 void __iomem *db = txr->tx_db.doorbell;
 426                 void *pdata = tx_push_buf->data;
 427                 u64 *end;
 428                 int j, push_len;
 429
 430                 /* Set COAL_NOW to be ready quickly for the next push */
 431                 tx_push->tx_bd_len_flags_type =
 432                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 433                                         TX_BD_TYPE_LONG_TX_BD |
 434                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 435                                         TX_BD_FLAGS_COAL_NOW |
 436                                         TX_BD_FLAGS_PACKET_END |
 437                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 438
 439                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 440                         tx_push1->tx_bd_hsize_lflags =
 441                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 442                 else
 443                         tx_push1->tx_bd_hsize_lflags = 0;
 444
 445                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 446                 tx_push1->tx_bd_cfa_action =
 447                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 448
 449                 end = pdata + length;
 450                 end = PTR_ALIGN(end, 8) - 1;
 451                 *end = 0;
 452
 453                 skb_copy_from_linear_data(skb, pdata, len);
 454                 pdata += len;
 455                 for (j = 0; j < last_frag; j++) {
 456                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 457                         void *fptr;
 458
 459                         fptr = skb_frag_address_safe(frag);
 460                         if (!fptr)
 461                                 goto normal_tx;
 462
 463                         memcpy(pdata, fptr, skb_frag_size(frag));
 464                         pdata += skb_frag_size(frag);
 465                 }
 466
 467                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 468                 txbd->tx_bd_haddr = txr->data_mapping;
 469                 prod = NEXT_TX(prod);
 470                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 471                 memcpy(txbd, tx_push1, sizeof(*txbd));
 472                 prod = NEXT_TX(prod);
 473                 tx_push->doorbell =
 474                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 475                 WRITE_ONCE(txr->tx_prod, prod);
 476
 477                 tx_buf->is_push = 1;
 478                 netdev_tx_sent_queue(txq, skb->len);
 479                 wmb();  /* Sync is_push and byte queue before pushing data */
 480
 481                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 482                 if (push_len > 16) {
 483                         __iowrite64_copy(db, tx_push_buf, 16);
 484                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 485                                          (push_len - 16) << 1);
 486                 } else {
 487                         __iowrite64_copy(db, tx_push_buf, push_len);
 488                 }
 489
 490                 goto tx_done;
 491         }
 492
 493 normal_tx:
 494         if (length < BNXT_MIN_PKT_SIZE) {
 495                 pad = BNXT_MIN_PKT_SIZE - length;
 496                 if (skb_pad(skb, pad))
 497                         /* SKB already freed. */
 498                         goto tx_kick_pending;
 499                 length = BNXT_MIN_PKT_SIZE;
 500         }
 501
 502         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 503
 504         if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 505                 goto tx_free;
 506
 507         dma_unmap_addr_set(tx_buf, mapping, mapping);
 508         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 509                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 510
 511         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 512
 513         prod = NEXT_TX(prod);
 514         txbd1 = (struct tx_bd_ext *)
 515                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 516
 517         txbd1->tx_bd_hsize_lflags = lflags;
 518         if (skb_is_gso(skb)) {
 519                 u32 hdr_len;
 520
 521                 if (skb->encapsulation)
 522                         hdr_len = skb_inner_tcp_all_headers(skb);
 523                 else
 524                         hdr_len = skb_tcp_all_headers(skb);
 525
 526                 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO |
 527                                         TX_BD_FLAGS_T_IPID |
 528                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 529                 length = skb_shinfo(skb)->gso_size;
 530                 txbd1->tx_bd_mss = cpu_to_le32(length);
 531                 length += hdr_len;
 532         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 533                 txbd1->tx_bd_hsize_lflags |=
 534                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 535                 txbd1->tx_bd_mss = 0;
 536         }
 537
 538         length >>= 9;
 539         if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
 540                 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
 541                                      skb->len);
 542                 i = 0;
 543                 goto tx_dma_error;
 544         }
 545         flags |= bnxt_lhint_arr[length];
 546         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 547
 548         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 549         txbd1->tx_bd_cfa_action =
 550                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 551         for (i = 0; i < last_frag; i++) {
 552                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 553
 554                 prod = NEXT_TX(prod);
 555                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 556
 557                 len = skb_frag_size(frag);
 558                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 559                                            DMA_TO_DEVICE);
 560
 561                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 562                         goto tx_dma_error;
 563
 564                 tx_buf = &txr->tx_buf_ring[prod];
 565                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 566
 567                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 568
 569                 flags = len << TX_BD_LEN_SHIFT;
 570                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 571         }
 572
 573         flags &= ~TX_BD_LEN;
 574         txbd->tx_bd_len_flags_type =
 575                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 576                             TX_BD_FLAGS_PACKET_END);
 577
 578         netdev_tx_sent_queue(txq, skb->len);
 579
 580         skb_tx_timestamp(skb);
 581
 582         /* Sync BD data before updating doorbell */
 583         wmb();
 584
 585         prod = NEXT_TX(prod);
 586         WRITE_ONCE(txr->tx_prod, prod);
 587
 588         if (!netdev_xmit_more() || netif_xmit_stopped(txq))
 589                 bnxt_txr_db_kick(bp, txr, prod);
 590         else
 591                 txr->kick_pending = 1;
 592
 593 tx_done:
 594
 595         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 596                 if (netdev_xmit_more() && !tx_buf->is_push)
 597                         bnxt_txr_db_kick(bp, txr, prod);
 598
 599                 netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
 600                                    bp->tx_wake_thresh);
 601         }
 602         return NETDEV_TX_OK;
 603
 604 tx_dma_error:
 605         if (BNXT_TX_PTP_IS_SET(lflags))
 606                 atomic_inc(&bp->ptp_cfg->tx_avail);
 607
 608         last_frag = i;
 609
 610         /* start back at beginning and unmap skb */
 611         prod = txr->tx_prod;
 612         tx_buf = &txr->tx_buf_ring[prod];
 613         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 614                          skb_headlen(skb), DMA_TO_DEVICE);
 615         prod = NEXT_TX(prod);
 616
 617         /* unmap remaining mapped pages */
 618         for (i = 0; i < last_frag; i++) {
 619                 prod = NEXT_TX(prod);
 620                 tx_buf = &txr->tx_buf_ring[prod];
 621                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 622                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 623                                DMA_TO_DEVICE);
 624         }
 625
 626 tx_free:
 627         dev_kfree_skb_any(skb);
 628 tx_kick_pending:
 629         if (txr->kick_pending)
 630                 bnxt_txr_db_kick(bp, txr, txr->tx_prod);
 631         txr->tx_buf_ring[txr->tx_prod].skb = NULL;
 632         dev_core_stats_tx_dropped_inc(dev);
 633         return NETDEV_TX_OK;
 634 }
 635
 636 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 637 {
 638         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 639         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 640         u16 cons = txr->tx_cons;
 641         struct pci_dev *pdev = bp->pdev;
 642         int i;
 643         unsigned int tx_bytes = 0;
 644
 645         for (i = 0; i < nr_pkts; i++) {
 646                 struct bnxt_sw_tx_bd *tx_buf;
 647                 struct sk_buff *skb;
 648                 int j, last;
 649
 650                 tx_buf = &txr->tx_buf_ring[cons];
 651                 cons = NEXT_TX(cons);
 652                 skb = tx_buf->skb;
 653                 tx_buf->skb = NULL;
 654
 655                 tx_bytes += skb->len;
 656
 657                 if (tx_buf->is_push) {
 658                         tx_buf->is_push = 0;
 659                         goto next_tx_int;
 660                 }
 661
 662                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 663                                  skb_headlen(skb), DMA_TO_DEVICE);
 664                 last = tx_buf->nr_frags;
 665
 666                 for (j = 0; j < last; j++) {
 667                         cons = NEXT_TX(cons);
 668                         tx_buf = &txr->tx_buf_ring[cons];
 669                         dma_unmap_page(
 670                                 &pdev->dev,
 671                                 dma_unmap_addr(tx_buf, mapping),
 672                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 673                                 DMA_TO_DEVICE);
 674                 }
 675                 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) {
 676                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
 677                                 /* PTP worker takes ownership of the skb */
 678                                 if (!bnxt_get_tx_ts_p5(bp, skb))
 679                                         skb = NULL;
 680                                 else
 681                                         atomic_inc(&bp->ptp_cfg->tx_avail);
 682                         }
 683                 }
 684
 685 next_tx_int:
 686                 cons = NEXT_TX(cons);
 687
 688                 dev_kfree_skb_any(skb);
 689         }
 690
 691         WRITE_ONCE(txr->tx_cons, cons);
 692
 693         __netif_txq_completed_wake(txq, nr_pkts, tx_bytes,
 694                                    bnxt_tx_avail(bp, txr), bp->tx_wake_thresh,
 695                                    READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING);
 696 }
 697
 698 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 699                                          struct bnxt_rx_ring_info *rxr,
 700                                          gfp_t gfp)
 701 {
 702         struct device *dev = &bp->pdev->dev;
 703         struct page *page;
 704
 705         page = page_pool_dev_alloc_pages(rxr->page_pool);
 706         if (!page)
 707                 return NULL;
 708
 709         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 710                                       DMA_ATTR_WEAK_ORDERING);
 711         if (dma_mapping_error(dev, *mapping)) {
 712                 page_pool_recycle_direct(rxr->page_pool, page);
 713                 return NULL;
 714         }
 715         return page;
 716 }
 717
 718 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping,
 719                                        gfp_t gfp)
 720 {
 721         u8 *data;
 722         struct pci_dev *pdev = bp->pdev;
 723
 724         if (gfp == GFP_ATOMIC)
 725                 data = napi_alloc_frag(bp->rx_buf_size);
 726         else
 727                 data = netdev_alloc_frag(bp->rx_buf_size);
 728         if (!data)
 729                 return NULL;
 730
 731         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 732                                         bp->rx_buf_use_size, bp->rx_dir,
 733                                         DMA_ATTR_WEAK_ORDERING);
 734
 735         if (dma_mapping_error(&pdev->dev, *mapping)) {
 736                 skb_free_frag(data);
 737                 data = NULL;
 738         }
 739         return data;
 740 }
 741
 742 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 743                        u16 prod, gfp_t gfp)
 744 {
 745         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 746         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 747         dma_addr_t mapping;
 748
 749         if (BNXT_RX_PAGE_MODE(bp)) {
 750                 struct page *page =
 751                         __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 752
 753                 if (!page)
 754                         return -ENOMEM;
 755
 756                 mapping += bp->rx_dma_offset;
 757                 rx_buf->data = page;
 758                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 759         } else {
 760                 u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp);
 761
 762                 if (!data)
 763                         return -ENOMEM;
 764
 765                 rx_buf->data = data;
 766                 rx_buf->data_ptr = data + bp->rx_offset;
 767         }
 768         rx_buf->mapping = mapping;
 769
 770         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 771         return 0;
 772 }
 773
 774 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 775 {
 776         u16 prod = rxr->rx_prod;
 777         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 778         struct rx_bd *cons_bd, *prod_bd;
 779
 780         prod_rx_buf = &rxr->rx_buf_ring[prod];
 781         cons_rx_buf = &rxr->rx_buf_ring[cons];
 782
 783         prod_rx_buf->data = data;
 784         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 785
 786         prod_rx_buf->mapping = cons_rx_buf->mapping;
 787
 788         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 789         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 790
 791         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 792 }
 793
 794 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 795 {
 796         u16 next, max = rxr->rx_agg_bmap_size;
 797
 798         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 799         if (next >= max)
 800                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 801         return next;
 802 }
 803
 804 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 805                                      struct bnxt_rx_ring_info *rxr,
 806                                      u16 prod, gfp_t gfp)
 807 {
 808         struct rx_bd *rxbd =
 809                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 810         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 811         struct pci_dev *pdev = bp->pdev;
 812         struct page *page;
 813         dma_addr_t mapping;
 814         u16 sw_prod = rxr->rx_sw_agg_prod;
 815         unsigned int offset = 0;
 816
 817         if (BNXT_RX_PAGE_MODE(bp)) {
 818                 page = __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 819
 820                 if (!page)
 821                         return -ENOMEM;
 822
 823         } else {
 824                 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 825                         page = rxr->rx_page;
 826                         if (!page) {
 827                                 page = alloc_page(gfp);
 828                                 if (!page)
 829                                         return -ENOMEM;
 830                                 rxr->rx_page = page;
 831                                 rxr->rx_page_offset = 0;
 832                         }
 833                         offset = rxr->rx_page_offset;
 834                         rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 835                         if (rxr->rx_page_offset == PAGE_SIZE)
 836                                 rxr->rx_page = NULL;
 837                         else
 838                                 get_page(page);
 839                 } else {
 840                         page = alloc_page(gfp);
 841                         if (!page)
 842                                 return -ENOMEM;
 843                 }
 844
 845                 mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 846                                              BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE,
 847                                              DMA_ATTR_WEAK_ORDERING);
 848                 if (dma_mapping_error(&pdev->dev, mapping)) {
 849                         __free_page(page);
 850                         return -EIO;
 851                 }
 852         }
 853
 854         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 855                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 856
 857         __set_bit(sw_prod, rxr->rx_agg_bmap);
 858         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 859         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 860
 861         rx_agg_buf->page = page;
 862         rx_agg_buf->offset = offset;
 863         rx_agg_buf->mapping = mapping;
 864         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 865         rxbd->rx_bd_opaque = sw_prod;
 866         return 0;
 867 }
 868
 869 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
 870                                        struct bnxt_cp_ring_info *cpr,
 871                                        u16 cp_cons, u16 curr)
 872 {
 873         struct rx_agg_cmp *agg;
 874
 875         cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
 876         agg = (struct rx_agg_cmp *)
 877                 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 878         return agg;
 879 }
 880
 881 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
 882                                               struct bnxt_rx_ring_info *rxr,
 883                                               u16 agg_id, u16 curr)
 884 {
 885         struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
 886
 887         return &tpa_info->agg_arr[curr];
 888 }
 889
 890 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
 891                                    u16 start, u32 agg_bufs, bool tpa)
 892 {
 893         struct bnxt_napi *bnapi = cpr->bnapi;
 894         struct bnxt *bp = bnapi->bp;
 895         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 896         u16 prod = rxr->rx_agg_prod;
 897         u16 sw_prod = rxr->rx_sw_agg_prod;
 898         bool p5_tpa = false;
 899         u32 i;
 900
 901         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
 902                 p5_tpa = true;
 903
 904         for (i = 0; i < agg_bufs; i++) {
 905                 u16 cons;
 906                 struct rx_agg_cmp *agg;
 907                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 908                 struct rx_bd *prod_bd;
 909                 struct page *page;
 910
 911                 if (p5_tpa)
 912                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
 913                 else
 914                         agg = bnxt_get_agg(bp, cpr, idx, start + i);
 915                 cons = agg->rx_agg_cmp_opaque;
 916                 __clear_bit(cons, rxr->rx_agg_bmap);
 917
 918                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 919                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 920
 921                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 922                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 923                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 924
 925                 /* It is possible for sw_prod to be equal to cons, so
 926                  * set cons_rx_buf->page to NULL first.
 927                  */
 928                 page = cons_rx_buf->page;
 929                 cons_rx_buf->page = NULL;
 930                 prod_rx_buf->page = page;
 931                 prod_rx_buf->offset = cons_rx_buf->offset;
 932
 933                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 934
 935                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 936
 937                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 938                 prod_bd->rx_bd_opaque = sw_prod;
 939
 940                 prod = NEXT_RX_AGG(prod);
 941                 sw_prod = NEXT_RX_AGG(sw_prod);
 942         }
 943         rxr->rx_agg_prod = prod;
 944         rxr->rx_sw_agg_prod = sw_prod;
 945 }
 946
 947 static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp,
 948                                               struct bnxt_rx_ring_info *rxr,
 949                                               u16 cons, void *data, u8 *data_ptr,
 950                                               dma_addr_t dma_addr,
 951                                               unsigned int offset_and_len)
 952 {
 953         unsigned int len = offset_and_len & 0xffff;
 954         struct page *page = data;
 955         u16 prod = rxr->rx_prod;
 956         struct sk_buff *skb;
 957         int err;
 958
 959         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 960         if (unlikely(err)) {
 961                 bnxt_reuse_rx_data(rxr, cons, data);
 962                 return NULL;
 963         }
 964         dma_addr -= bp->rx_dma_offset;
 965         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 966                              DMA_ATTR_WEAK_ORDERING);
 967         skb = build_skb(page_address(page), PAGE_SIZE);
 968         if (!skb) {
 969                 page_pool_recycle_direct(rxr->page_pool, page);
 970                 return NULL;
 971         }
 972         skb_mark_for_recycle(skb);
 973         skb_reserve(skb, bp->rx_dma_offset);
 974         __skb_put(skb, len);
 975
 976         return skb;
 977 }
 978
 979 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 980                                         struct bnxt_rx_ring_info *rxr,
 981                                         u16 cons, void *data, u8 *data_ptr,
 982                                         dma_addr_t dma_addr,
 983                                         unsigned int offset_and_len)
 984 {
 985         unsigned int payload = offset_and_len >> 16;
 986         unsigned int len = offset_and_len & 0xffff;
 987         skb_frag_t *frag;
 988         struct page *page = data;
 989         u16 prod = rxr->rx_prod;
 990         struct sk_buff *skb;
 991         int off, err;
 992
 993         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 994         if (unlikely(err)) {
 995                 bnxt_reuse_rx_data(rxr, cons, data);
 996                 return NULL;
 997         }
 998         dma_addr -= bp->rx_dma_offset;
 999         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
1000                              DMA_ATTR_WEAK_ORDERING);
1001
1002         if (unlikely(!payload))
1003                 payload = eth_get_headlen(bp->dev, data_ptr, len);
1004
1005         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
1006         if (!skb) {
1007                 page_pool_recycle_direct(rxr->page_pool, page);
1008                 return NULL;
1009         }
1010
1011         skb_mark_for_recycle(skb);
1012         off = (void *)data_ptr - page_address(page);
1013         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
1014         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
1015                payload + NET_IP_ALIGN);
1016
1017         frag = &skb_shinfo(skb)->frags[0];
1018         skb_frag_size_sub(frag, payload);
1019         skb_frag_off_add(frag, payload);
1020         skb->data_len -= payload;
1021         skb->tail += payload;
1022
1023         return skb;
1024 }
1025
1026 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
1027                                    struct bnxt_rx_ring_info *rxr, u16 cons,
1028                                    void *data, u8 *data_ptr,
1029                                    dma_addr_t dma_addr,
1030                                    unsigned int offset_and_len)
1031 {
1032         u16 prod = rxr->rx_prod;
1033         struct sk_buff *skb;
1034         int err;
1035
1036         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1037         if (unlikely(err)) {
1038                 bnxt_reuse_rx_data(rxr, cons, data);
1039                 return NULL;
1040         }
1041
1042         skb = build_skb(data, bp->rx_buf_size);
1043         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
1044                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
1045         if (!skb) {
1046                 skb_free_frag(data);
1047                 return NULL;
1048         }
1049
1050         skb_reserve(skb, bp->rx_offset);
1051         skb_put(skb, offset_and_len & 0xffff);
1052         return skb;
1053 }
1054
1055 static u32 __bnxt_rx_agg_pages(struct bnxt *bp,
1056                                struct bnxt_cp_ring_info *cpr,
1057                                struct skb_shared_info *shinfo,
1058                                u16 idx, u32 agg_bufs, bool tpa,
1059                                struct xdp_buff *xdp)
1060 {
1061         struct bnxt_napi *bnapi = cpr->bnapi;
1062         struct pci_dev *pdev = bp->pdev;
1063         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1064         u16 prod = rxr->rx_agg_prod;
1065         u32 i, total_frag_len = 0;
1066         bool p5_tpa = false;
1067
1068         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1069                 p5_tpa = true;
1070
1071         for (i = 0; i < agg_bufs; i++) {
1072                 skb_frag_t *frag = &shinfo->frags[i];
1073                 u16 cons, frag_len;
1074                 struct rx_agg_cmp *agg;
1075                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1076                 struct page *page;
1077                 dma_addr_t mapping;
1078
1079                 if (p5_tpa)
1080                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1081                 else
1082                         agg = bnxt_get_agg(bp, cpr, idx, i);
1083                 cons = agg->rx_agg_cmp_opaque;
1084                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1085                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1086
1087                 cons_rx_buf = &rxr->rx_agg_ring[cons];
1088                 skb_frag_off_set(frag, cons_rx_buf->offset);
1089                 skb_frag_size_set(frag, frag_len);
1090                 __skb_frag_set_page(frag, cons_rx_buf->page);
1091                 shinfo->nr_frags = i + 1;
1092                 __clear_bit(cons, rxr->rx_agg_bmap);
1093
1094                 /* It is possible for bnxt_alloc_rx_page() to allocate
1095                  * a sw_prod index that equals the cons index, so we
1096                  * need to clear the cons entry now.
1097                  */
1098                 mapping = cons_rx_buf->mapping;
1099                 page = cons_rx_buf->page;
1100                 cons_rx_buf->page = NULL;
1101
1102                 if (xdp && page_is_pfmemalloc(page))
1103                         xdp_buff_set_frag_pfmemalloc(xdp);
1104
1105                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1106                         unsigned int nr_frags;
1107
1108                         nr_frags = --shinfo->nr_frags;
1109                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1110                         cons_rx_buf->page = page;
1111
1112                         /* Update prod since possibly some pages have been
1113                          * allocated already.
1114                          */
1115                         rxr->rx_agg_prod = prod;
1116                         bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1117                         return 0;
1118                 }
1119
1120                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1121                                      bp->rx_dir,
1122                                      DMA_ATTR_WEAK_ORDERING);
1123
1124                 total_frag_len += frag_len;
1125                 prod = NEXT_RX_AGG(prod);
1126         }
1127         rxr->rx_agg_prod = prod;
1128         return total_frag_len;
1129 }
1130
1131 static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp,
1132                                              struct bnxt_cp_ring_info *cpr,
1133                                              struct sk_buff *skb, u16 idx,
1134                                              u32 agg_bufs, bool tpa)
1135 {
1136         struct skb_shared_info *shinfo = skb_shinfo(skb);
1137         u32 total_frag_len = 0;
1138
1139         total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx,
1140                                              agg_bufs, tpa, NULL);
1141         if (!total_frag_len) {
1142                 dev_kfree_skb(skb);
1143                 return NULL;
1144         }
1145
1146         skb->data_len += total_frag_len;
1147         skb->len += total_frag_len;
1148         skb->truesize += PAGE_SIZE * agg_bufs;
1149         return skb;
1150 }
1151
1152 static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp,
1153                                  struct bnxt_cp_ring_info *cpr,
1154                                  struct xdp_buff *xdp, u16 idx,
1155                                  u32 agg_bufs, bool tpa)
1156 {
1157         struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp);
1158         u32 total_frag_len = 0;
1159
1160         if (!xdp_buff_has_frags(xdp))
1161                 shinfo->nr_frags = 0;
1162
1163         total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo,
1164                                              idx, agg_bufs, tpa, xdp);
1165         if (total_frag_len) {
1166                 xdp_buff_set_frags_flag(xdp);
1167                 shinfo->nr_frags = agg_bufs;
1168                 shinfo->xdp_frags_size = total_frag_len;
1169         }
1170         return total_frag_len;
1171 }
1172
1173 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1174                                u8 agg_bufs, u32 *raw_cons)
1175 {
1176         u16 last;
1177         struct rx_agg_cmp *agg;
1178
1179         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1180         last = RING_CMP(*raw_cons);
1181         agg = (struct rx_agg_cmp *)
1182                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1183         return RX_AGG_CMP_VALID(agg, *raw_cons);
1184 }
1185
1186 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1187                                             unsigned int len,
1188                                             dma_addr_t mapping)
1189 {
1190         struct bnxt *bp = bnapi->bp;
1191         struct pci_dev *pdev = bp->pdev;
1192         struct sk_buff *skb;
1193
1194         skb = napi_alloc_skb(&bnapi->napi, len);
1195         if (!skb)
1196                 return NULL;
1197
1198         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1199                                 bp->rx_dir);
1200
1201         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1202                len + NET_IP_ALIGN);
1203
1204         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1205                                    bp->rx_dir);
1206
1207         skb_put(skb, len);
1208         return skb;
1209 }
1210
1211 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1212                            u32 *raw_cons, void *cmp)
1213 {
1214         struct rx_cmp *rxcmp = cmp;
1215         u32 tmp_raw_cons = *raw_cons;
1216         u8 cmp_type, agg_bufs = 0;
1217
1218         cmp_type = RX_CMP_TYPE(rxcmp);
1219
1220         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1221                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1222                             RX_CMP_AGG_BUFS) >>
1223                            RX_CMP_AGG_BUFS_SHIFT;
1224         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1225                 struct rx_tpa_end_cmp *tpa_end = cmp;
1226
1227                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1228                         return 0;
1229
1230                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1231         }
1232
1233         if (agg_bufs) {
1234                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1235                         return -EBUSY;
1236         }
1237         *raw_cons = tmp_raw_cons;
1238         return 0;
1239 }
1240
1241 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1242 {
1243         if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
1244                 return;
1245
1246         if (BNXT_PF(bp))
1247                 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1248         else
1249                 schedule_delayed_work(&bp->fw_reset_task, delay);
1250 }
1251
1252 static void bnxt_queue_sp_work(struct bnxt *bp)
1253 {
1254         if (BNXT_PF(bp))
1255                 queue_work(bnxt_pf_wq, &bp->sp_task);
1256         else
1257                 schedule_work(&bp->sp_task);
1258 }
1259
1260 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1261 {
1262         if (!rxr->bnapi->in_reset) {
1263                 rxr->bnapi->in_reset = true;
1264                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1265                         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1266                 else
1267                         set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
1268                 bnxt_queue_sp_work(bp);
1269         }
1270         rxr->rx_next_cons = 0xffff;
1271 }
1272
1273 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1274 {
1275         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1276         u16 idx = agg_id & MAX_TPA_P5_MASK;
1277
1278         if (test_bit(idx, map->agg_idx_bmap))
1279                 idx = find_first_zero_bit(map->agg_idx_bmap,
1280                                           BNXT_AGG_IDX_BMAP_SIZE);
1281         __set_bit(idx, map->agg_idx_bmap);
1282         map->agg_id_tbl[agg_id] = idx;
1283         return idx;
1284 }
1285
1286 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1287 {
1288         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1289
1290         __clear_bit(idx, map->agg_idx_bmap);
1291 }
1292
1293 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1294 {
1295         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1296
1297         return map->agg_id_tbl[agg_id];
1298 }
1299
1300 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1301                            struct rx_tpa_start_cmp *tpa_start,
1302                            struct rx_tpa_start_cmp_ext *tpa_start1)
1303 {
1304         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1305         struct bnxt_tpa_info *tpa_info;
1306         u16 cons, prod, agg_id;
1307         struct rx_bd *prod_bd;
1308         dma_addr_t mapping;
1309
1310         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1311                 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1312                 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1313         } else {
1314                 agg_id = TPA_START_AGG_ID(tpa_start);
1315         }
1316         cons = tpa_start->rx_tpa_start_cmp_opaque;
1317         prod = rxr->rx_prod;
1318         cons_rx_buf = &rxr->rx_buf_ring[cons];
1319         prod_rx_buf = &rxr->rx_buf_ring[prod];
1320         tpa_info = &rxr->rx_tpa[agg_id];
1321
1322         if (unlikely(cons != rxr->rx_next_cons ||
1323                      TPA_START_ERROR(tpa_start))) {
1324                 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1325                             cons, rxr->rx_next_cons,
1326                             TPA_START_ERROR_CODE(tpa_start1));
1327                 bnxt_sched_reset(bp, rxr);
1328                 return;
1329         }
1330         /* Store cfa_code in tpa_info to use in tpa_end
1331          * completion processing.
1332          */
1333         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1334         prod_rx_buf->data = tpa_info->data;
1335         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1336
1337         mapping = tpa_info->mapping;
1338         prod_rx_buf->mapping = mapping;
1339
1340         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1341
1342         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1343
1344         tpa_info->data = cons_rx_buf->data;
1345         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1346         cons_rx_buf->data = NULL;
1347         tpa_info->mapping = cons_rx_buf->mapping;
1348
1349         tpa_info->len =
1350                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1351                                 RX_TPA_START_CMP_LEN_SHIFT;
1352         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1353                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1354
1355                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1356                 tpa_info->gso_type = SKB_GSO_TCPV4;
1357                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1358                 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1359                         tpa_info->gso_type = SKB_GSO_TCPV6;
1360                 tpa_info->rss_hash =
1361                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1362         } else {
1363                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1364                 tpa_info->gso_type = 0;
1365                 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n");
1366         }
1367         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1368         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1369         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1370         tpa_info->agg_count = 0;
1371
1372         rxr->rx_prod = NEXT_RX(prod);
1373         cons = NEXT_RX(cons);
1374         rxr->rx_next_cons = NEXT_RX(cons);
1375         cons_rx_buf = &rxr->rx_buf_ring[cons];
1376
1377         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1378         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1379         cons_rx_buf->data = NULL;
1380 }
1381
1382 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1383 {
1384         if (agg_bufs)
1385                 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1386 }
1387
1388 #ifdef CONFIG_INET
1389 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1390 {
1391         struct udphdr *uh = NULL;
1392
1393         if (ip_proto == htons(ETH_P_IP)) {
1394                 struct iphdr *iph = (struct iphdr *)skb->data;
1395
1396                 if (iph->protocol == IPPROTO_UDP)
1397                         uh = (struct udphdr *)(iph + 1);
1398         } else {
1399                 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1400
1401                 if (iph->nexthdr == IPPROTO_UDP)
1402                         uh = (struct udphdr *)(iph + 1);
1403         }
1404         if (uh) {
1405                 if (uh->check)
1406                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1407                 else
1408                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1409         }
1410 }
1411 #endif
1412
1413 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1414                                            int payload_off, int tcp_ts,
1415                                            struct sk_buff *skb)
1416 {
1417 #ifdef CONFIG_INET
1418         struct tcphdr *th;
1419         int len, nw_off;
1420         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1421         u32 hdr_info = tpa_info->hdr_info;
1422         bool loopback = false;
1423
1424         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1425         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1426         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1427
1428         /* If the packet is an internal loopback packet, the offsets will
1429          * have an extra 4 bytes.
1430          */
1431         if (inner_mac_off == 4) {
1432                 loopback = true;
1433         } else if (inner_mac_off > 4) {
1434                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1435                                             ETH_HLEN - 2));
1436
1437                 /* We only support inner iPv4/ipv6.  If we don't see the
1438                  * correct protocol ID, it must be a loopback packet where
1439                  * the offsets are off by 4.
1440                  */
1441                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1442                         loopback = true;
1443         }
1444         if (loopback) {
1445                 /* internal loopback packet, subtract all offsets by 4 */
1446                 inner_ip_off -= 4;
1447                 inner_mac_off -= 4;
1448                 outer_ip_off -= 4;
1449         }
1450
1451         nw_off = inner_ip_off - ETH_HLEN;
1452         skb_set_network_header(skb, nw_off);
1453         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1454                 struct ipv6hdr *iph = ipv6_hdr(skb);
1455
1456                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1457                 len = skb->len - skb_transport_offset(skb);
1458                 th = tcp_hdr(skb);
1459                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1460         } else {
1461                 struct iphdr *iph = ip_hdr(skb);
1462
1463                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1464                 len = skb->len - skb_transport_offset(skb);
1465                 th = tcp_hdr(skb);
1466                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1467         }
1468
1469         if (inner_mac_off) { /* tunnel */
1470                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1471                                             ETH_HLEN - 2));
1472
1473                 bnxt_gro_tunnel(skb, proto);
1474         }
1475 #endif
1476         return skb;
1477 }
1478
1479 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1480                                            int payload_off, int tcp_ts,
1481                                            struct sk_buff *skb)
1482 {
1483 #ifdef CONFIG_INET
1484         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1485         u32 hdr_info = tpa_info->hdr_info;
1486         int iphdr_len, nw_off;
1487
1488         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1489         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1490         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1491
1492         nw_off = inner_ip_off - ETH_HLEN;
1493         skb_set_network_header(skb, nw_off);
1494         iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1495                      sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1496         skb_set_transport_header(skb, nw_off + iphdr_len);
1497
1498         if (inner_mac_off) { /* tunnel */
1499                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1500                                             ETH_HLEN - 2));
1501
1502                 bnxt_gro_tunnel(skb, proto);
1503         }
1504 #endif
1505         return skb;
1506 }
1507
1508 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1509 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1510
1511 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1512                                            int payload_off, int tcp_ts,
1513                                            struct sk_buff *skb)
1514 {
1515 #ifdef CONFIG_INET
1516         struct tcphdr *th;
1517         int len, nw_off, tcp_opt_len = 0;
1518
1519         if (tcp_ts)
1520                 tcp_opt_len = 12;
1521
1522         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1523                 struct iphdr *iph;
1524
1525                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1526                          ETH_HLEN;
1527                 skb_set_network_header(skb, nw_off);
1528                 iph = ip_hdr(skb);
1529                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1530                 len = skb->len - skb_transport_offset(skb);
1531                 th = tcp_hdr(skb);
1532                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1533         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1534                 struct ipv6hdr *iph;
1535
1536                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1537                          ETH_HLEN;
1538                 skb_set_network_header(skb, nw_off);
1539                 iph = ipv6_hdr(skb);
1540                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1541                 len = skb->len - skb_transport_offset(skb);
1542                 th = tcp_hdr(skb);
1543                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1544         } else {
1545                 dev_kfree_skb_any(skb);
1546                 return NULL;
1547         }
1548
1549         if (nw_off) /* tunnel */
1550                 bnxt_gro_tunnel(skb, skb->protocol);
1551 #endif
1552         return skb;
1553 }
1554
1555 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1556                                            struct bnxt_tpa_info *tpa_info,
1557                                            struct rx_tpa_end_cmp *tpa_end,
1558                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1559                                            struct sk_buff *skb)
1560 {
1561 #ifdef CONFIG_INET
1562         int payload_off;
1563         u16 segs;
1564
1565         segs = TPA_END_TPA_SEGS(tpa_end);
1566         if (segs == 1)
1567                 return skb;
1568
1569         NAPI_GRO_CB(skb)->count = segs;
1570         skb_shinfo(skb)->gso_size =
1571                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1572         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1573         if (bp->flags & BNXT_FLAG_CHIP_P5)
1574                 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1575         else
1576                 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1577         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1578         if (likely(skb))
1579                 tcp_gro_complete(skb);
1580 #endif
1581         return skb;
1582 }
1583
1584 /* Given the cfa_code of a received packet determine which
1585  * netdev (vf-rep or PF) the packet is destined to.
1586  */
1587 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1588 {
1589         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1590
1591         /* if vf-rep dev is NULL, the must belongs to the PF */
1592         return dev ? dev : bp->dev;
1593 }
1594
1595 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1596                                            struct bnxt_cp_ring_info *cpr,
1597                                            u32 *raw_cons,
1598                                            struct rx_tpa_end_cmp *tpa_end,
1599                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1600                                            u8 *event)
1601 {
1602         struct bnxt_napi *bnapi = cpr->bnapi;
1603         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1604         u8 *data_ptr, agg_bufs;
1605         unsigned int len;
1606         struct bnxt_tpa_info *tpa_info;
1607         dma_addr_t mapping;
1608         struct sk_buff *skb;
1609         u16 idx = 0, agg_id;
1610         void *data;
1611         bool gro;
1612
1613         if (unlikely(bnapi->in_reset)) {
1614                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1615
1616                 if (rc < 0)
1617                         return ERR_PTR(-EBUSY);
1618                 return NULL;
1619         }
1620
1621         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1622                 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1623                 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1624                 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1625                 tpa_info = &rxr->rx_tpa[agg_id];
1626                 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1627                         netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1628                                     agg_bufs, tpa_info->agg_count);
1629                         agg_bufs = tpa_info->agg_count;
1630                 }
1631                 tpa_info->agg_count = 0;
1632                 *event |= BNXT_AGG_EVENT;
1633                 bnxt_free_agg_idx(rxr, agg_id);
1634                 idx = agg_id;
1635                 gro = !!(bp->flags & BNXT_FLAG_GRO);
1636         } else {
1637                 agg_id = TPA_END_AGG_ID(tpa_end);
1638                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1639                 tpa_info = &rxr->rx_tpa[agg_id];
1640                 idx = RING_CMP(*raw_cons);
1641                 if (agg_bufs) {
1642                         if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1643                                 return ERR_PTR(-EBUSY);
1644
1645                         *event |= BNXT_AGG_EVENT;
1646                         idx = NEXT_CMP(idx);
1647                 }
1648                 gro = !!TPA_END_GRO(tpa_end);
1649         }
1650         data = tpa_info->data;
1651         data_ptr = tpa_info->data_ptr;
1652         prefetch(data_ptr);
1653         len = tpa_info->len;
1654         mapping = tpa_info->mapping;
1655
1656         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1657                 bnxt_abort_tpa(cpr, idx, agg_bufs);
1658                 if (agg_bufs > MAX_SKB_FRAGS)
1659                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1660                                     agg_bufs, (int)MAX_SKB_FRAGS);
1661                 return NULL;
1662         }
1663
1664         if (len <= bp->rx_copy_thresh) {
1665                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1666                 if (!skb) {
1667                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1668                         cpr->sw_stats.rx.rx_oom_discards += 1;
1669                         return NULL;
1670                 }
1671         } else {
1672                 u8 *new_data;
1673                 dma_addr_t new_mapping;
1674
1675                 new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC);
1676                 if (!new_data) {
1677                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1678                         cpr->sw_stats.rx.rx_oom_discards += 1;
1679                         return NULL;
1680                 }
1681
1682                 tpa_info->data = new_data;
1683                 tpa_info->data_ptr = new_data + bp->rx_offset;
1684                 tpa_info->mapping = new_mapping;
1685
1686                 skb = build_skb(data, bp->rx_buf_size);
1687                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1688                                        bp->rx_buf_use_size, bp->rx_dir,
1689                                        DMA_ATTR_WEAK_ORDERING);
1690
1691                 if (!skb) {
1692                         skb_free_frag(data);
1693                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1694                         cpr->sw_stats.rx.rx_oom_discards += 1;
1695                         return NULL;
1696                 }
1697                 skb_reserve(skb, bp->rx_offset);
1698                 skb_put(skb, len);
1699         }
1700
1701         if (agg_bufs) {
1702                 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true);
1703                 if (!skb) {
1704                         /* Page reuse already handled by bnxt_rx_pages(). */
1705                         cpr->sw_stats.rx.rx_oom_discards += 1;
1706                         return NULL;
1707                 }
1708         }
1709
1710         skb->protocol =
1711                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1712
1713         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1714                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1715
1716         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1717             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1718                 __be16 vlan_proto = htons(tpa_info->metadata >>
1719                                           RX_CMP_FLAGS2_METADATA_TPID_SFT);
1720                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1721
1722                 if (eth_type_vlan(vlan_proto)) {
1723                         __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1724                 } else {
1725                         dev_kfree_skb(skb);
1726                         return NULL;
1727                 }
1728         }
1729
1730         skb_checksum_none_assert(skb);
1731         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1732                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1733                 skb->csum_level =
1734                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1735         }
1736
1737         if (gro)
1738                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1739
1740         return skb;
1741 }
1742
1743 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1744                          struct rx_agg_cmp *rx_agg)
1745 {
1746         u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1747         struct bnxt_tpa_info *tpa_info;
1748
1749         agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1750         tpa_info = &rxr->rx_tpa[agg_id];
1751         BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1752         tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1753 }
1754
1755 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1756                              struct sk_buff *skb)
1757 {
1758         if (skb->dev != bp->dev) {
1759                 /* this packet belongs to a vf-rep */
1760                 bnxt_vf_rep_rx(bp, skb);
1761                 return;
1762         }
1763         skb_record_rx_queue(skb, bnapi->index);
1764         napi_gro_receive(&bnapi->napi, skb);
1765 }
1766
1767 /* returns the following:
1768  * 1       - 1 packet successfully received
1769  * 0       - successful TPA_START, packet not completed yet
1770  * -EBUSY  - completion ring does not have all the agg buffers yet
1771  * -ENOMEM - packet aborted due to out of memory
1772  * -EIO    - packet aborted due to hw error indicated in BD
1773  */
1774 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1775                        u32 *raw_cons, u8 *event)
1776 {
1777         struct bnxt_napi *bnapi = cpr->bnapi;
1778         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1779         struct net_device *dev = bp->dev;
1780         struct rx_cmp *rxcmp;
1781         struct rx_cmp_ext *rxcmp1;
1782         u32 tmp_raw_cons = *raw_cons;
1783         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1784         struct bnxt_sw_rx_bd *rx_buf;
1785         unsigned int len;
1786         u8 *data_ptr, agg_bufs, cmp_type;
1787         bool xdp_active = false;
1788         dma_addr_t dma_addr;
1789         struct sk_buff *skb;
1790         struct xdp_buff xdp;
1791         u32 flags, misc;
1792         void *data;
1793         int rc = 0;
1794
1795         rxcmp = (struct rx_cmp *)
1796                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1797
1798         cmp_type = RX_CMP_TYPE(rxcmp);
1799
1800         if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1801                 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1802                 goto next_rx_no_prod_no_len;
1803         }
1804
1805         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1806         cp_cons = RING_CMP(tmp_raw_cons);
1807         rxcmp1 = (struct rx_cmp_ext *)
1808                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1809
1810         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1811                 return -EBUSY;
1812
1813         /* The valid test of the entry must be done first before
1814          * reading any further.
1815          */
1816         dma_rmb();
1817         prod = rxr->rx_prod;
1818
1819         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1820                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1821                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1822
1823                 *event |= BNXT_RX_EVENT;
1824                 goto next_rx_no_prod_no_len;
1825
1826         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1827                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1828                                    (struct rx_tpa_end_cmp *)rxcmp,
1829                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1830
1831                 if (IS_ERR(skb))
1832                         return -EBUSY;
1833
1834                 rc = -ENOMEM;
1835                 if (likely(skb)) {
1836                         bnxt_deliver_skb(bp, bnapi, skb);
1837                         rc = 1;
1838                 }
1839                 *event |= BNXT_RX_EVENT;
1840                 goto next_rx_no_prod_no_len;
1841         }
1842
1843         cons = rxcmp->rx_cmp_opaque;
1844         if (unlikely(cons != rxr->rx_next_cons)) {
1845                 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp);
1846
1847                 /* 0xffff is forced error, don't print it */
1848                 if (rxr->rx_next_cons != 0xffff)
1849                         netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1850                                     cons, rxr->rx_next_cons);
1851                 bnxt_sched_reset(bp, rxr);
1852                 if (rc1)
1853                         return rc1;
1854                 goto next_rx_no_prod_no_len;
1855         }
1856         rx_buf = &rxr->rx_buf_ring[cons];
1857         data = rx_buf->data;
1858         data_ptr = rx_buf->data_ptr;
1859         prefetch(data_ptr);
1860
1861         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1862         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1863
1864         if (agg_bufs) {
1865                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1866                         return -EBUSY;
1867
1868                 cp_cons = NEXT_CMP(cp_cons);
1869                 *event |= BNXT_AGG_EVENT;
1870         }
1871         *event |= BNXT_RX_EVENT;
1872
1873         rx_buf->data = NULL;
1874         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1875                 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1876
1877                 bnxt_reuse_rx_data(rxr, cons, data);
1878                 if (agg_bufs)
1879                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1880                                                false);
1881
1882                 rc = -EIO;
1883                 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1884                         bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1885                         if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
1886                             !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
1887                                 netdev_warn_once(bp->dev, "RX buffer error %x\n",
1888                                                  rx_err);
1889                                 bnxt_sched_reset(bp, rxr);
1890                         }
1891                 }
1892                 goto next_rx_no_len;
1893         }
1894
1895         flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type);
1896         len = flags >> RX_CMP_LEN_SHIFT;
1897         dma_addr = rx_buf->mapping;
1898
1899         if (bnxt_xdp_attached(bp, rxr)) {
1900                 bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp);
1901                 if (agg_bufs) {
1902                         u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp,
1903                                                              cp_cons, agg_bufs,
1904                                                              false);
1905                         if (!frag_len) {
1906                                 cpr->sw_stats.rx.rx_oom_discards += 1;
1907                                 rc = -ENOMEM;
1908                                 goto next_rx;
1909                         }
1910                 }
1911                 xdp_active = true;
1912         }
1913
1914         if (xdp_active) {
1915                 if (bnxt_rx_xdp(bp, rxr, cons, xdp, data, &data_ptr, &len, event)) {
1916                         rc = 1;
1917                         goto next_rx;
1918                 }
1919         }
1920
1921         if (len <= bp->rx_copy_thresh) {
1922                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1923                 bnxt_reuse_rx_data(rxr, cons, data);
1924                 if (!skb) {
1925                         if (agg_bufs) {
1926                                 if (!xdp_active)
1927                                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1928                                                                agg_bufs, false);
1929                                 else
1930                                         bnxt_xdp_buff_frags_free(rxr, &xdp);
1931                         }
1932                         cpr->sw_stats.rx.rx_oom_discards += 1;
1933                         rc = -ENOMEM;
1934                         goto next_rx;
1935                 }
1936         } else {
1937                 u32 payload;
1938
1939                 if (rx_buf->data_ptr == data_ptr)
1940                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1941                 else
1942                         payload = 0;
1943                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1944                                       payload | len);
1945                 if (!skb) {
1946                         cpr->sw_stats.rx.rx_oom_discards += 1;
1947                         rc = -ENOMEM;
1948                         goto next_rx;
1949                 }
1950         }
1951
1952         if (agg_bufs) {
1953                 if (!xdp_active) {
1954                         skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false);
1955                         if (!skb) {
1956                                 cpr->sw_stats.rx.rx_oom_discards += 1;
1957                                 rc = -ENOMEM;
1958                                 goto next_rx;
1959                         }
1960                 } else {
1961                         skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1);
1962                         if (!skb) {
1963                                 /* we should be able to free the old skb here */
1964                                 bnxt_xdp_buff_frags_free(rxr, &xdp);
1965                                 cpr->sw_stats.rx.rx_oom_discards += 1;
1966                                 rc = -ENOMEM;
1967                                 goto next_rx;
1968                         }
1969                 }
1970         }
1971
1972         if (RX_CMP_HASH_VALID(rxcmp)) {
1973                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1974                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1975
1976                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1977                 if (hash_type != 1 && hash_type != 3)
1978                         type = PKT_HASH_TYPE_L3;
1979                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1980         }
1981
1982         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1983         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1984
1985         if ((rxcmp1->rx_cmp_flags2 &
1986              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1987             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1988                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1989                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1990                 __be16 vlan_proto = htons(meta_data >>
1991                                           RX_CMP_FLAGS2_METADATA_TPID_SFT);
1992
1993                 if (eth_type_vlan(vlan_proto)) {
1994                         __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1995                 } else {
1996                         dev_kfree_skb(skb);
1997                         goto next_rx;
1998                 }
1999         }
2000
2001         skb_checksum_none_assert(skb);
2002         if (RX_CMP_L4_CS_OK(rxcmp1)) {
2003                 if (dev->features & NETIF_F_RXCSUM) {
2004                         skb->ip_summed = CHECKSUM_UNNECESSARY;
2005                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
2006                 }
2007         } else {
2008                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
2009                         if (dev->features & NETIF_F_RXCSUM)
2010                                 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
2011                 }
2012         }
2013
2014         if (unlikely((flags & RX_CMP_FLAGS_ITYPES_MASK) ==
2015                      RX_CMP_FLAGS_ITYPE_PTP_W_TS) || bp->ptp_all_rx_tstamp) {
2016                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
2017                         u32 cmpl_ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp);
2018                         u64 ns, ts;
2019
2020                         if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) {
2021                                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2022
2023                                 spin_lock_bh(&ptp->ptp_lock);
2024                                 ns = timecounter_cyc2time(&ptp->tc, ts);
2025                                 spin_unlock_bh(&ptp->ptp_lock);
2026                                 memset(skb_hwtstamps(skb), 0,
2027                                        sizeof(*skb_hwtstamps(skb)));
2028                                 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
2029                         }
2030                 }
2031         }
2032         bnxt_deliver_skb(bp, bnapi, skb);
2033         rc = 1;
2034
2035 next_rx:
2036         cpr->rx_packets += 1;
2037         cpr->rx_bytes += len;
2038
2039 next_rx_no_len:
2040         rxr->rx_prod = NEXT_RX(prod);
2041         rxr->rx_next_cons = NEXT_RX(cons);
2042
2043 next_rx_no_prod_no_len:
2044         *raw_cons = tmp_raw_cons;
2045
2046         return rc;
2047 }
2048
2049 /* In netpoll mode, if we are using a combined completion ring, we need to
2050  * discard the rx packets and recycle the buffers.
2051  */
2052 static int bnxt_force_rx_discard(struct bnxt *bp,
2053                                  struct bnxt_cp_ring_info *cpr,
2054                                  u32 *raw_cons, u8 *event)
2055 {
2056         u32 tmp_raw_cons = *raw_cons;
2057         struct rx_cmp_ext *rxcmp1;
2058         struct rx_cmp *rxcmp;
2059         u16 cp_cons;
2060         u8 cmp_type;
2061         int rc;
2062
2063         cp_cons = RING_CMP(tmp_raw_cons);
2064         rxcmp = (struct rx_cmp *)
2065                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2066
2067         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2068         cp_cons = RING_CMP(tmp_raw_cons);
2069         rxcmp1 = (struct rx_cmp_ext *)
2070                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2071
2072         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2073                 return -EBUSY;
2074
2075         /* The valid test of the entry must be done first before
2076          * reading any further.
2077          */
2078         dma_rmb();
2079         cmp_type = RX_CMP_TYPE(rxcmp);
2080         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
2081                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2082                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2083         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2084                 struct rx_tpa_end_cmp_ext *tpa_end1;
2085
2086                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
2087                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
2088                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
2089         }
2090         rc = bnxt_rx_pkt(bp, cpr, raw_cons, event);
2091         if (rc && rc != -EBUSY)
2092                 cpr->sw_stats.rx.rx_netpoll_discards += 1;
2093         return rc;
2094 }
2095
2096 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
2097 {
2098         struct bnxt_fw_health *fw_health = bp->fw_health;
2099         u32 reg = fw_health->regs[reg_idx];
2100         u32 reg_type, reg_off, val = 0;
2101
2102         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
2103         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
2104         switch (reg_type) {
2105         case BNXT_FW_HEALTH_REG_TYPE_CFG:
2106                 pci_read_config_dword(bp->pdev, reg_off, &val);
2107                 break;
2108         case BNXT_FW_HEALTH_REG_TYPE_GRC:
2109                 reg_off = fw_health->mapped_regs[reg_idx];
2110                 fallthrough;
2111         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
2112                 val = readl(bp->bar0 + reg_off);
2113                 break;
2114         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
2115                 val = readl(bp->bar1 + reg_off);
2116                 break;
2117         }
2118         if (reg_idx == BNXT_FW_RESET_INPROG_REG)
2119                 val &= fw_health->fw_reset_inprog_reg_mask;
2120         return val;
2121 }
2122
2123 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
2124 {
2125         int i;
2126
2127         for (i = 0; i < bp->rx_nr_rings; i++) {
2128                 u16 grp_idx = bp->rx_ring[i].bnapi->index;
2129                 struct bnxt_ring_grp_info *grp_info;
2130
2131                 grp_info = &bp->grp_info[grp_idx];
2132                 if (grp_info->agg_fw_ring_id == ring_id)
2133                         return grp_idx;
2134         }
2135         return INVALID_HW_RING_ID;
2136 }
2137
2138 static void bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2)
2139 {
2140         u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1);
2141
2142         switch (err_type) {
2143         case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL:
2144                 netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n",
2145                            BNXT_EVENT_INVALID_SIGNAL_DATA(data2));
2146                 break;
2147         case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM:
2148                 netdev_warn(bp->dev, "Pause Storm detected!\n");
2149                 break;
2150         case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD:
2151                 netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n");
2152                 break;
2153         default:
2154                 netdev_err(bp->dev, "FW reported unknown error type %u\n",
2155                            err_type);
2156                 break;
2157         }
2158 }
2159
2160 #define BNXT_GET_EVENT_PORT(data)       \
2161         ((data) &                       \
2162          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
2163
2164 #define BNXT_EVENT_RING_TYPE(data2)     \
2165         ((data2) &                      \
2166          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
2167
2168 #define BNXT_EVENT_RING_TYPE_RX(data2)  \
2169         (BNXT_EVENT_RING_TYPE(data2) == \
2170          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
2171
2172 #define BNXT_EVENT_PHC_EVENT_TYPE(data1)        \
2173         (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\
2174          ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT)
2175
2176 #define BNXT_EVENT_PHC_RTC_UPDATE(data1)        \
2177         (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\
2178          ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT)
2179
2180 #define BNXT_PHC_BITS   48
2181
2182 static int bnxt_async_event_process(struct bnxt *bp,
2183                                     struct hwrm_async_event_cmpl *cmpl)
2184 {
2185         u16 event_id = le16_to_cpu(cmpl->event_id);
2186         u32 data1 = le32_to_cpu(cmpl->event_data1);
2187         u32 data2 = le32_to_cpu(cmpl->event_data2);
2188
2189         netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n",
2190                    event_id, data1, data2);
2191
2192         /* TODO CHIMP_FW: Define event id's for link change, error etc */
2193         switch (event_id) {
2194         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
2195                 struct bnxt_link_info *link_info = &bp->link_info;
2196
2197                 if (BNXT_VF(bp))
2198                         goto async_event_process_exit;
2199
2200                 /* print unsupported speed warning in forced speed mode only */
2201                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
2202                     (data1 & 0x20000)) {
2203                         u16 fw_speed = link_info->force_link_speed;
2204                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
2205
2206                         if (speed != SPEED_UNKNOWN)
2207                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
2208                                             speed);
2209                 }
2210                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
2211         }
2212                 fallthrough;
2213         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
2214         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
2215                 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2216                 fallthrough;
2217         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2218                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2219                 break;
2220         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2221                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2222                 break;
2223         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2224                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
2225
2226                 if (BNXT_VF(bp))
2227                         break;
2228
2229                 if (bp->pf.port_id != port_id)
2230                         break;
2231
2232                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2233                 break;
2234         }
2235         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2236                 if (BNXT_PF(bp))
2237                         goto async_event_process_exit;
2238                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2239                 break;
2240         case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2241                 char *type_str = "Solicited";
2242
2243                 if (!bp->fw_health)
2244                         goto async_event_process_exit;
2245
2246                 bp->fw_reset_timestamp = jiffies;
2247                 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2248                 if (!bp->fw_reset_min_dsecs)
2249                         bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2250                 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2251                 if (!bp->fw_reset_max_dsecs)
2252                         bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2253                 if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) {
2254                         set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state);
2255                 } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2256                         type_str = "Fatal";
2257                         bp->fw_health->fatalities++;
2258                         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2259                 } else if (data2 && BNXT_FW_STATUS_HEALTHY !=
2260                            EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) {
2261                         type_str = "Non-fatal";
2262                         bp->fw_health->survivals++;
2263                         set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
2264                 }
2265                 netif_warn(bp, hw, bp->dev,
2266                            "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n",
2267                            type_str, data1, data2,
2268                            bp->fw_reset_min_dsecs * 100,
2269                            bp->fw_reset_max_dsecs * 100);
2270                 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2271                 break;
2272         }
2273         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2274                 struct bnxt_fw_health *fw_health = bp->fw_health;
2275                 char *status_desc = "healthy";
2276                 u32 status;
2277
2278                 if (!fw_health)
2279                         goto async_event_process_exit;
2280
2281                 if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) {
2282                         fw_health->enabled = false;
2283                         netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n");
2284                         break;
2285                 }
2286                 fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2287                 fw_health->tmr_multiplier =
2288                         DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2289                                      bp->current_interval * 10);
2290                 fw_health->tmr_counter = fw_health->tmr_multiplier;
2291                 if (!fw_health->enabled)
2292                         fw_health->last_fw_heartbeat =
2293                                 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2294                 fw_health->last_fw_reset_cnt =
2295                         bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2296                 status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
2297                 if (status != BNXT_FW_STATUS_HEALTHY)
2298                         status_desc = "unhealthy";
2299                 netif_info(bp, drv, bp->dev,
2300                            "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n",
2301                            fw_health->primary ? "primary" : "backup", status,
2302                            status_desc, fw_health->last_fw_reset_cnt);
2303                 if (!fw_health->enabled) {
2304                         /* Make sure tmr_counter is set and visible to
2305                          * bnxt_health_check() before setting enabled to true.
2306                          */
2307                         smp_wmb();
2308                         fw_health->enabled = true;
2309                 }
2310                 goto async_event_process_exit;
2311         }
2312         case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION:
2313                 netif_notice(bp, hw, bp->dev,
2314                              "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n",
2315                              data1, data2);
2316                 goto async_event_process_exit;
2317         case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2318                 struct bnxt_rx_ring_info *rxr;
2319                 u16 grp_idx;
2320
2321                 if (bp->flags & BNXT_FLAG_CHIP_P5)
2322                         goto async_event_process_exit;
2323
2324                 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2325                             BNXT_EVENT_RING_TYPE(data2), data1);
2326                 if (!BNXT_EVENT_RING_TYPE_RX(data2))
2327                         goto async_event_process_exit;
2328
2329                 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2330                 if (grp_idx == INVALID_HW_RING_ID) {
2331                         netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2332                                     data1);
2333                         goto async_event_process_exit;
2334                 }
2335                 rxr = bp->bnapi[grp_idx]->rx_ring;
2336                 bnxt_sched_reset(bp, rxr);
2337                 goto async_event_process_exit;
2338         }
2339         case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: {
2340                 struct bnxt_fw_health *fw_health = bp->fw_health;
2341
2342                 netif_notice(bp, hw, bp->dev,
2343                              "Received firmware echo request, data1: 0x%x, data2: 0x%x\n",
2344                              data1, data2);
2345                 if (fw_health) {
2346                         fw_health->echo_req_data1 = data1;
2347                         fw_health->echo_req_data2 = data2;
2348                         set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event);
2349                         break;
2350                 }
2351                 goto async_event_process_exit;
2352         }
2353         case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: {
2354                 bnxt_ptp_pps_event(bp, data1, data2);
2355                 goto async_event_process_exit;
2356         }
2357         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: {
2358                 bnxt_event_error_report(bp, data1, data2);
2359                 goto async_event_process_exit;
2360         }
2361         case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: {
2362                 switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) {
2363                 case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE:
2364                         if (BNXT_PTP_USE_RTC(bp)) {
2365                                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2366                                 u64 ns;
2367
2368                                 if (!ptp)
2369                                         goto async_event_process_exit;
2370
2371                                 spin_lock_bh(&ptp->ptp_lock);
2372                                 bnxt_ptp_update_current_time(bp);
2373                                 ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) <<
2374                                        BNXT_PHC_BITS) | ptp->current_time);
2375                                 bnxt_ptp_rtc_timecounter_init(ptp, ns);
2376                                 spin_unlock_bh(&ptp->ptp_lock);
2377                         }
2378                         break;
2379                 }
2380                 goto async_event_process_exit;
2381         }
2382         case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: {
2383                 u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff;
2384
2385                 hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED);
2386                 goto async_event_process_exit;
2387         }
2388         default:
2389                 goto async_event_process_exit;
2390         }
2391         bnxt_queue_sp_work(bp);
2392 async_event_process_exit:
2393         return 0;
2394 }
2395
2396 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2397 {
2398         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2399         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2400         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2401                                 (struct hwrm_fwd_req_cmpl *)txcmp;
2402
2403         switch (cmpl_type) {
2404         case CMPL_BASE_TYPE_HWRM_DONE:
2405                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2406                 hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE);
2407                 break;
2408
2409         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2410                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2411
2412                 if ((vf_id < bp->pf.first_vf_id) ||
2413                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2414                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2415                                    vf_id);
2416                         return -EINVAL;
2417                 }
2418
2419                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2420                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2421                 bnxt_queue_sp_work(bp);
2422                 break;
2423
2424         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2425                 bnxt_async_event_process(bp,
2426                                          (struct hwrm_async_event_cmpl *)txcmp);
2427                 break;
2428
2429         default:
2430                 break;
2431         }
2432
2433         return 0;
2434 }
2435
2436 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2437 {
2438         struct bnxt_napi *bnapi = dev_instance;
2439         struct bnxt *bp = bnapi->bp;
2440         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2441         u32 cons = RING_CMP(cpr->cp_raw_cons);
2442
2443         cpr->event_ctr++;
2444         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2445         napi_schedule(&bnapi->napi);
2446         return IRQ_HANDLED;
2447 }
2448
2449 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2450 {
2451         u32 raw_cons = cpr->cp_raw_cons;
2452         u16 cons = RING_CMP(raw_cons);
2453         struct tx_cmp *txcmp;
2454
2455         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2456
2457         return TX_CMP_VALID(txcmp, raw_cons);
2458 }
2459
2460 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2461 {
2462         struct bnxt_napi *bnapi = dev_instance;
2463         struct bnxt *bp = bnapi->bp;
2464         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2465         u32 cons = RING_CMP(cpr->cp_raw_cons);
2466         u32 int_status;
2467
2468         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2469
2470         if (!bnxt_has_work(bp, cpr)) {
2471                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2472                 /* return if erroneous interrupt */
2473                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2474                         return IRQ_NONE;
2475         }
2476
2477         /* disable ring IRQ */
2478         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2479
2480         /* Return here if interrupt is shared and is disabled. */
2481         if (unlikely(atomic_read(&bp->intr_sem) != 0))
2482                 return IRQ_HANDLED;
2483
2484         napi_schedule(&bnapi->napi);
2485         return IRQ_HANDLED;
2486 }
2487
2488 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2489                             int budget)
2490 {
2491         struct bnxt_napi *bnapi = cpr->bnapi;
2492         u32 raw_cons = cpr->cp_raw_cons;
2493         u32 cons;
2494         int tx_pkts = 0;
2495         int rx_pkts = 0;
2496         u8 event = 0;
2497         struct tx_cmp *txcmp;
2498
2499         cpr->has_more_work = 0;
2500         cpr->had_work_done = 1;
2501         while (1) {
2502                 int rc;
2503
2504                 cons = RING_CMP(raw_cons);
2505                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2506
2507                 if (!TX_CMP_VALID(txcmp, raw_cons))
2508                         break;
2509
2510                 /* The valid test of the entry must be done first before
2511                  * reading any further.
2512                  */
2513                 dma_rmb();
2514                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2515                         tx_pkts++;
2516                         /* return full budget so NAPI will complete. */
2517                         if (unlikely(tx_pkts >= bp->tx_wake_thresh)) {
2518                                 rx_pkts = budget;
2519                                 raw_cons = NEXT_RAW_CMP(raw_cons);
2520                                 if (budget)
2521                                         cpr->has_more_work = 1;
2522                                 break;
2523                         }
2524                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2525                         if (likely(budget))
2526                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2527                         else
2528                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2529                                                            &event);
2530                         if (likely(rc >= 0))
2531                                 rx_pkts += rc;
2532                         /* Increment rx_pkts when rc is -ENOMEM to count towards
2533                          * the NAPI budget.  Otherwise, we may potentially loop
2534                          * here forever if we consistently cannot allocate
2535                          * buffers.
2536                          */
2537                         else if (rc == -ENOMEM && budget)
2538                                 rx_pkts++;
2539                         else if (rc == -EBUSY)  /* partial completion */
2540                                 break;
2541                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2542                                      CMPL_BASE_TYPE_HWRM_DONE) ||
2543                                     (TX_CMP_TYPE(txcmp) ==
2544                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2545                                     (TX_CMP_TYPE(txcmp) ==
2546                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2547                         bnxt_hwrm_handler(bp, txcmp);
2548                 }
2549                 raw_cons = NEXT_RAW_CMP(raw_cons);
2550
2551                 if (rx_pkts && rx_pkts == budget) {
2552                         cpr->has_more_work = 1;
2553                         break;
2554                 }
2555         }
2556
2557         if (event & BNXT_REDIRECT_EVENT)
2558                 xdp_do_flush();
2559
2560         if (event & BNXT_TX_EVENT) {
2561                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2562                 u16 prod = txr->tx_prod;
2563
2564                 /* Sync BD data before updating doorbell */
2565                 wmb();
2566
2567                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2568         }
2569
2570         cpr->cp_raw_cons = raw_cons;
2571         bnapi->tx_pkts += tx_pkts;
2572         bnapi->events |= event;
2573         return rx_pkts;
2574 }
2575
2576 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2577 {
2578         if (bnapi->tx_pkts) {
2579                 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2580                 bnapi->tx_pkts = 0;
2581         }
2582
2583         if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2584                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2585
2586                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2587         }
2588         if (bnapi->events & BNXT_AGG_EVENT) {
2589                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2590
2591                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2592         }
2593         bnapi->events = 0;
2594 }
2595
2596 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2597                           int budget)
2598 {
2599         struct bnxt_napi *bnapi = cpr->bnapi;
2600         int rx_pkts;
2601
2602         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2603
2604         /* ACK completion ring before freeing tx ring and producing new
2605          * buffers in rx/agg rings to prevent overflowing the completion
2606          * ring.
2607          */
2608         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2609
2610         __bnxt_poll_work_done(bp, bnapi);
2611         return rx_pkts;
2612 }
2613
2614 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2615 {
2616         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2617         struct bnxt *bp = bnapi->bp;
2618         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2619         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2620         struct tx_cmp *txcmp;
2621         struct rx_cmp_ext *rxcmp1;
2622         u32 cp_cons, tmp_raw_cons;
2623         u32 raw_cons = cpr->cp_raw_cons;
2624         u32 rx_pkts = 0;
2625         u8 event = 0;
2626
2627         while (1) {
2628                 int rc;
2629
2630                 cp_cons = RING_CMP(raw_cons);
2631                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2632
2633                 if (!TX_CMP_VALID(txcmp, raw_cons))
2634                         break;
2635
2636                 /* The valid test of the entry must be done first before
2637                  * reading any further.
2638                  */
2639                 dma_rmb();
2640                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2641                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2642                         cp_cons = RING_CMP(tmp_raw_cons);
2643                         rxcmp1 = (struct rx_cmp_ext *)
2644                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2645
2646                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2647                                 break;
2648
2649                         /* force an error to recycle the buffer */
2650                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2651                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2652
2653                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2654                         if (likely(rc == -EIO) && budget)
2655                                 rx_pkts++;
2656                         else if (rc == -EBUSY)  /* partial completion */
2657                                 break;
2658                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2659                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2660                         bnxt_hwrm_handler(bp, txcmp);
2661                 } else {
2662                         netdev_err(bp->dev,
2663                                    "Invalid completion received on special ring\n");
2664                 }
2665                 raw_cons = NEXT_RAW_CMP(raw_cons);
2666
2667                 if (rx_pkts == budget)
2668                         break;
2669         }
2670
2671         cpr->cp_raw_cons = raw_cons;
2672         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2673         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2674
2675         if (event & BNXT_AGG_EVENT)
2676                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2677
2678         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2679                 napi_complete_done(napi, rx_pkts);
2680                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2681         }
2682         return rx_pkts;
2683 }
2684
2685 static int bnxt_poll(struct napi_struct *napi, int budget)
2686 {
2687         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2688         struct bnxt *bp = bnapi->bp;
2689         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2690         int work_done = 0;
2691
2692         if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2693                 napi_complete(napi);
2694                 return 0;
2695         }
2696         while (1) {
2697                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2698
2699                 if (work_done >= budget) {
2700                         if (!budget)
2701                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2702                         break;
2703                 }
2704
2705                 if (!bnxt_has_work(bp, cpr)) {
2706                         if (napi_complete_done(napi, work_done))
2707                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2708                         break;
2709                 }
2710         }
2711         if (bp->flags & BNXT_FLAG_DIM) {
2712                 struct dim_sample dim_sample = {};
2713
2714                 dim_update_sample(cpr->event_ctr,
2715                                   cpr->rx_packets,
2716                                   cpr->rx_bytes,
2717                                   &dim_sample);
2718                 net_dim(&cpr->dim, dim_sample);
2719         }
2720         return work_done;
2721 }
2722
2723 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2724 {
2725         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2726         int i, work_done = 0;
2727
2728         for (i = 0; i < 2; i++) {
2729                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2730
2731                 if (cpr2) {
2732                         work_done += __bnxt_poll_work(bp, cpr2,
2733                                                       budget - work_done);
2734                         cpr->has_more_work |= cpr2->has_more_work;
2735                 }
2736         }
2737         return work_done;
2738 }
2739
2740 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2741                                  u64 dbr_type)
2742 {
2743         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2744         int i;
2745
2746         for (i = 0; i < 2; i++) {
2747                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2748                 struct bnxt_db_info *db;
2749
2750                 if (cpr2 && cpr2->had_work_done) {
2751                         db = &cpr2->cp_db;
2752                         bnxt_writeq(bp, db->db_key64 | dbr_type |
2753                                     RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2754                         cpr2->had_work_done = 0;
2755                 }
2756         }
2757         __bnxt_poll_work_done(bp, bnapi);
2758 }
2759
2760 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2761 {
2762         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2763         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2764         struct bnxt_cp_ring_info *cpr_rx;
2765         u32 raw_cons = cpr->cp_raw_cons;
2766         struct bnxt *bp = bnapi->bp;
2767         struct nqe_cn *nqcmp;
2768         int work_done = 0;
2769         u32 cons;
2770
2771         if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2772                 napi_complete(napi);
2773                 return 0;
2774         }
2775         if (cpr->has_more_work) {
2776                 cpr->has_more_work = 0;
2777                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2778         }
2779         while (1) {
2780                 cons = RING_CMP(raw_cons);
2781                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2782
2783                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2784                         if (cpr->has_more_work)
2785                                 break;
2786
2787                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL);
2788                         cpr->cp_raw_cons = raw_cons;
2789                         if (napi_complete_done(napi, work_done))
2790                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2791                                                   cpr->cp_raw_cons);
2792                         goto poll_done;
2793                 }
2794
2795                 /* The valid test of the entry must be done first before
2796                  * reading any further.
2797                  */
2798                 dma_rmb();
2799
2800                 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2801                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2802                         struct bnxt_cp_ring_info *cpr2;
2803
2804                         /* No more budget for RX work */
2805                         if (budget && work_done >= budget && idx == BNXT_RX_HDL)
2806                                 break;
2807
2808                         cpr2 = cpr->cp_ring_arr[idx];
2809                         work_done += __bnxt_poll_work(bp, cpr2,
2810                                                       budget - work_done);
2811                         cpr->has_more_work |= cpr2->has_more_work;
2812                 } else {
2813                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2814                 }
2815                 raw_cons = NEXT_RAW_CMP(raw_cons);
2816         }
2817         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ);
2818         if (raw_cons != cpr->cp_raw_cons) {
2819                 cpr->cp_raw_cons = raw_cons;
2820                 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2821         }
2822 poll_done:
2823         cpr_rx = cpr->cp_ring_arr[BNXT_RX_HDL];
2824         if (cpr_rx && (bp->flags & BNXT_FLAG_DIM)) {
2825                 struct dim_sample dim_sample = {};
2826
2827                 dim_update_sample(cpr->event_ctr,
2828                                   cpr_rx->rx_packets,
2829                                   cpr_rx->rx_bytes,
2830                                   &dim_sample);
2831                 net_dim(&cpr->dim, dim_sample);
2832         }
2833         return work_done;
2834 }
2835
2836 static void bnxt_free_tx_skbs(struct bnxt *bp)
2837 {
2838         int i, max_idx;
2839         struct pci_dev *pdev = bp->pdev;
2840
2841         if (!bp->tx_ring)
2842                 return;
2843
2844         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2845         for (i = 0; i < bp->tx_nr_rings; i++) {
2846                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2847                 int j;
2848
2849                 if (!txr->tx_buf_ring)
2850                         continue;
2851
2852                 for (j = 0; j < max_idx;) {
2853                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2854                         struct sk_buff *skb;
2855                         int k, last;
2856
2857                         if (i < bp->tx_nr_rings_xdp &&
2858                             tx_buf->action == XDP_REDIRECT) {
2859                                 dma_unmap_single(&pdev->dev,
2860                                         dma_unmap_addr(tx_buf, mapping),
2861                                         dma_unmap_len(tx_buf, len),
2862                                         DMA_TO_DEVICE);
2863                                 xdp_return_frame(tx_buf->xdpf);
2864                                 tx_buf->action = 0;
2865                                 tx_buf->xdpf = NULL;
2866                                 j++;
2867                                 continue;
2868                         }
2869
2870                         skb = tx_buf->skb;
2871                         if (!skb) {
2872                                 j++;
2873                                 continue;
2874                         }
2875
2876                         tx_buf->skb = NULL;
2877
2878                         if (tx_buf->is_push) {
2879                                 dev_kfree_skb(skb);
2880                                 j += 2;
2881                                 continue;
2882                         }
2883
2884                         dma_unmap_single(&pdev->dev,
2885                                          dma_unmap_addr(tx_buf, mapping),
2886                                          skb_headlen(skb),
2887                                          DMA_TO_DEVICE);
2888
2889                         last = tx_buf->nr_frags;
2890                         j += 2;
2891                         for (k = 0; k < last; k++, j++) {
2892                                 int ring_idx = j & bp->tx_ring_mask;
2893                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2894
2895                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2896                                 dma_unmap_page(
2897                                         &pdev->dev,
2898                                         dma_unmap_addr(tx_buf, mapping),
2899                                         skb_frag_size(frag), DMA_TO_DEVICE);
2900                         }
2901                         dev_kfree_skb(skb);
2902                 }
2903                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2904         }
2905 }
2906
2907 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
2908 {
2909         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
2910         struct pci_dev *pdev = bp->pdev;
2911         struct bnxt_tpa_idx_map *map;
2912         int i, max_idx, max_agg_idx;
2913
2914         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2915         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2916         if (!rxr->rx_tpa)
2917                 goto skip_rx_tpa_free;
2918
2919         for (i = 0; i < bp->max_tpa; i++) {
2920                 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
2921                 u8 *data = tpa_info->data;
2922
2923                 if (!data)
2924                         continue;
2925
2926                 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
2927                                        bp->rx_buf_use_size, bp->rx_dir,
2928                                        DMA_ATTR_WEAK_ORDERING);
2929
2930                 tpa_info->data = NULL;
2931
2932                 skb_free_frag(data);
2933         }
2934
2935 skip_rx_tpa_free:
2936         if (!rxr->rx_buf_ring)
2937                 goto skip_rx_buf_free;
2938
2939         for (i = 0; i < max_idx; i++) {
2940                 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
2941                 dma_addr_t mapping = rx_buf->mapping;
2942                 void *data = rx_buf->data;
2943
2944                 if (!data)
2945                         continue;
2946
2947                 rx_buf->data = NULL;
2948                 if (BNXT_RX_PAGE_MODE(bp)) {
2949                         mapping -= bp->rx_dma_offset;
2950                         dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE,
2951                                              bp->rx_dir,
2952                                              DMA_ATTR_WEAK_ORDERING);
2953                         page_pool_recycle_direct(rxr->page_pool, data);
2954                 } else {
2955                         dma_unmap_single_attrs(&pdev->dev, mapping,
2956                                                bp->rx_buf_use_size, bp->rx_dir,
2957                                                DMA_ATTR_WEAK_ORDERING);
2958                         skb_free_frag(data);
2959                 }
2960         }
2961
2962 skip_rx_buf_free:
2963         if (!rxr->rx_agg_ring)
2964                 goto skip_rx_agg_free;
2965
2966         for (i = 0; i < max_agg_idx; i++) {
2967                 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
2968                 struct page *page = rx_agg_buf->page;
2969
2970                 if (!page)
2971                         continue;
2972
2973                 if (BNXT_RX_PAGE_MODE(bp)) {
2974                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2975                                              BNXT_RX_PAGE_SIZE, bp->rx_dir,
2976                                              DMA_ATTR_WEAK_ORDERING);
2977                         rx_agg_buf->page = NULL;
2978                         __clear_bit(i, rxr->rx_agg_bmap);
2979
2980                         page_pool_recycle_direct(rxr->page_pool, page);
2981                 } else {
2982                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2983                                              BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE,
2984                                              DMA_ATTR_WEAK_ORDERING);
2985                         rx_agg_buf->page = NULL;
2986                         __clear_bit(i, rxr->rx_agg_bmap);
2987
2988                         __free_page(page);
2989                 }
2990         }
2991
2992 skip_rx_agg_free:
2993         if (rxr->rx_page) {
2994                 __free_page(rxr->rx_page);
2995                 rxr->rx_page = NULL;
2996         }
2997         map = rxr->rx_tpa_idx_map;
2998         if (map)
2999                 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
3000 }
3001
3002 static void bnxt_free_rx_skbs(struct bnxt *bp)
3003 {
3004         int i;
3005
3006         if (!bp->rx_ring)
3007                 return;
3008
3009         for (i = 0; i < bp->rx_nr_rings; i++)
3010                 bnxt_free_one_rx_ring_skbs(bp, i);
3011 }
3012
3013 static void bnxt_free_skbs(struct bnxt *bp)
3014 {
3015         bnxt_free_tx_skbs(bp);
3016         bnxt_free_rx_skbs(bp);
3017 }
3018
3019 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len)
3020 {
3021         u8 init_val = mem_init->init_val;
3022         u16 offset = mem_init->offset;
3023         u8 *p2 = p;
3024         int i;
3025
3026         if (!init_val)
3027                 return;
3028         if (offset == BNXT_MEM_INVALID_OFFSET) {
3029                 memset(p, init_val, len);
3030                 return;
3031         }
3032         for (i = 0; i < len; i += mem_init->size)
3033                 *(p2 + i + offset) = init_val;
3034 }
3035
3036 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3037 {
3038         struct pci_dev *pdev = bp->pdev;
3039         int i;
3040
3041         if (!rmem->pg_arr)
3042                 goto skip_pages;
3043
3044         for (i = 0; i < rmem->nr_pages; i++) {
3045                 if (!rmem->pg_arr[i])
3046                         continue;
3047
3048                 dma_free_coherent(&pdev->dev, rmem->page_size,
3049                                   rmem->pg_arr[i], rmem->dma_arr[i]);
3050
3051                 rmem->pg_arr[i] = NULL;
3052         }
3053 skip_pages:
3054         if (rmem->pg_tbl) {
3055                 size_t pg_tbl_size = rmem->nr_pages * 8;
3056
3057                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3058                         pg_tbl_size = rmem->page_size;
3059                 dma_free_coherent(&pdev->dev, pg_tbl_size,
3060                                   rmem->pg_tbl, rmem->pg_tbl_map);
3061                 rmem->pg_tbl = NULL;
3062         }
3063         if (rmem->vmem_size && *rmem->vmem) {
3064                 vfree(*rmem->vmem);
3065                 *rmem->vmem = NULL;
3066         }
3067 }
3068
3069 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3070 {
3071         struct pci_dev *pdev = bp->pdev;
3072         u64 valid_bit = 0;
3073         int i;
3074
3075         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
3076                 valid_bit = PTU_PTE_VALID;
3077         if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
3078                 size_t pg_tbl_size = rmem->nr_pages * 8;
3079
3080                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3081                         pg_tbl_size = rmem->page_size;
3082                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
3083                                                   &rmem->pg_tbl_map,
3084                                                   GFP_KERNEL);
3085                 if (!rmem->pg_tbl)
3086                         return -ENOMEM;
3087         }
3088
3089         for (i = 0; i < rmem->nr_pages; i++) {
3090                 u64 extra_bits = valid_bit;
3091
3092                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
3093                                                      rmem->page_size,
3094                                                      &rmem->dma_arr[i],
3095                                                      GFP_KERNEL);
3096                 if (!rmem->pg_arr[i])
3097                         return -ENOMEM;
3098
3099                 if (rmem->mem_init)
3100                         bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i],
3101                                           rmem->page_size);
3102                 if (rmem->nr_pages > 1 || rmem->depth > 0) {
3103                         if (i == rmem->nr_pages - 2 &&
3104                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3105                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
3106                         else if (i == rmem->nr_pages - 1 &&
3107                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3108                                 extra_bits |= PTU_PTE_LAST;
3109                         rmem->pg_tbl[i] =
3110                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
3111                 }
3112         }
3113
3114         if (rmem->vmem_size) {
3115                 *rmem->vmem = vzalloc(rmem->vmem_size);
3116                 if (!(*rmem->vmem))
3117                         return -ENOMEM;
3118         }
3119         return 0;
3120 }
3121
3122 static void bnxt_free_tpa_info(struct bnxt *bp)
3123 {
3124         int i, j;
3125
3126         for (i = 0; i < bp->rx_nr_rings; i++) {
3127                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3128
3129                 kfree(rxr->rx_tpa_idx_map);
3130                 rxr->rx_tpa_idx_map = NULL;
3131                 if (rxr->rx_tpa) {
3132                         for (j = 0; j < bp->max_tpa; j++) {
3133                                 kfree(rxr->rx_tpa[j].agg_arr);
3134                                 rxr->rx_tpa[j].agg_arr = NULL;
3135                         }
3136                 }
3137                 kfree(rxr->rx_tpa);
3138                 rxr->rx_tpa = NULL;
3139         }
3140 }
3141
3142 static int bnxt_alloc_tpa_info(struct bnxt *bp)
3143 {
3144         int i, j;
3145
3146         bp->max_tpa = MAX_TPA;
3147         if (bp->flags & BNXT_FLAG_CHIP_P5) {
3148                 if (!bp->max_tpa_v2)
3149                         return 0;
3150                 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
3151         }
3152
3153         for (i = 0; i < bp->rx_nr_rings; i++) {
3154                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3155                 struct rx_agg_cmp *agg;
3156
3157                 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
3158                                       GFP_KERNEL);
3159                 if (!rxr->rx_tpa)
3160                         return -ENOMEM;
3161
3162                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3163                         continue;
3164                 for (j = 0; j < bp->max_tpa; j++) {
3165                         agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL);
3166                         if (!agg)
3167                                 return -ENOMEM;
3168                         rxr->rx_tpa[j].agg_arr = agg;
3169                 }
3170                 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
3171                                               GFP_KERNEL);
3172                 if (!rxr->rx_tpa_idx_map)
3173                         return -ENOMEM;
3174         }
3175         return 0;
3176 }
3177
3178 static void bnxt_free_rx_rings(struct bnxt *bp)
3179 {
3180         int i;
3181
3182         if (!bp->rx_ring)
3183                 return;
3184
3185         bnxt_free_tpa_info(bp);
3186         for (i = 0; i < bp->rx_nr_rings; i++) {
3187                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3188                 struct bnxt_ring_struct *ring;
3189
3190                 if (rxr->xdp_prog)
3191                         bpf_prog_put(rxr->xdp_prog);
3192
3193                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
3194                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
3195
3196                 page_pool_destroy(rxr->page_pool);
3197                 rxr->page_pool = NULL;
3198
3199                 kfree(rxr->rx_agg_bmap);
3200                 rxr->rx_agg_bmap = NULL;
3201
3202                 ring = &rxr->rx_ring_struct;
3203                 bnxt_free_ring(bp, &ring->ring_mem);
3204
3205                 ring = &rxr->rx_agg_ring_struct;
3206                 bnxt_free_ring(bp, &ring->ring_mem);
3207         }
3208 }
3209
3210 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
3211                                    struct bnxt_rx_ring_info *rxr)
3212 {
3213         struct page_pool_params pp = { 0 };
3214
3215         pp.pool_size = bp->rx_ring_size;
3216         pp.nid = dev_to_node(&bp->pdev->dev);
3217         pp.napi = &rxr->bnapi->napi;
3218         pp.dev = &bp->pdev->dev;
3219         pp.dma_dir = DMA_BIDIRECTIONAL;
3220
3221         rxr->page_pool = page_pool_create(&pp);
3222         if (IS_ERR(rxr->page_pool)) {
3223                 int err = PTR_ERR(rxr->page_pool);
3224
3225                 rxr->page_pool = NULL;
3226                 return err;
3227         }
3228         return 0;
3229 }
3230
3231 static int bnxt_alloc_rx_rings(struct bnxt *bp)
3232 {
3233         int i, rc = 0, agg_rings = 0;
3234
3235         if (!bp->rx_ring)
3236                 return -ENOMEM;
3237
3238         if (bp->flags & BNXT_FLAG_AGG_RINGS)
3239                 agg_rings = 1;
3240
3241         for (i = 0; i < bp->rx_nr_rings; i++) {
3242                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3243                 struct bnxt_ring_struct *ring;
3244
3245                 ring = &rxr->rx_ring_struct;
3246
3247                 rc = bnxt_alloc_rx_page_pool(bp, rxr);
3248                 if (rc)
3249                         return rc;
3250
3251                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
3252                 if (rc < 0)
3253                         return rc;
3254
3255                 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
3256                                                 MEM_TYPE_PAGE_POOL,
3257                                                 rxr->page_pool);
3258                 if (rc) {
3259                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
3260                         return rc;
3261                 }
3262
3263                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3264                 if (rc)
3265                         return rc;
3266
3267                 ring->grp_idx = i;
3268                 if (agg_rings) {
3269                         u16 mem_size;
3270
3271                         ring = &rxr->rx_agg_ring_struct;
3272                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3273                         if (rc)
3274                                 return rc;
3275
3276                         ring->grp_idx = i;
3277                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
3278                         mem_size = rxr->rx_agg_bmap_size / 8;
3279                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
3280                         if (!rxr->rx_agg_bmap)
3281                                 return -ENOMEM;
3282                 }
3283         }
3284         if (bp->flags & BNXT_FLAG_TPA)
3285                 rc = bnxt_alloc_tpa_info(bp);
3286         return rc;
3287 }
3288
3289 static void bnxt_free_tx_rings(struct bnxt *bp)
3290 {
3291         int i;
3292         struct pci_dev *pdev = bp->pdev;
3293
3294         if (!bp->tx_ring)
3295                 return;
3296
3297         for (i = 0; i < bp->tx_nr_rings; i++) {
3298                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3299                 struct bnxt_ring_struct *ring;
3300
3301                 if (txr->tx_push) {
3302                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
3303                                           txr->tx_push, txr->tx_push_mapping);
3304                         txr->tx_push = NULL;
3305                 }
3306
3307                 ring = &txr->tx_ring_struct;
3308
3309                 bnxt_free_ring(bp, &ring->ring_mem);
3310         }
3311 }
3312
3313 static int bnxt_alloc_tx_rings(struct bnxt *bp)
3314 {
3315         int i, j, rc;
3316         struct pci_dev *pdev = bp->pdev;
3317
3318         bp->tx_push_size = 0;
3319         if (bp->tx_push_thresh) {
3320                 int push_size;
3321
3322                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
3323                                         bp->tx_push_thresh);
3324
3325                 if (push_size > 256) {
3326                         push_size = 0;
3327                         bp->tx_push_thresh = 0;
3328                 }
3329
3330                 bp->tx_push_size = push_size;
3331         }
3332
3333         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
3334                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3335                 struct bnxt_ring_struct *ring;
3336                 u8 qidx;
3337
3338                 ring = &txr->tx_ring_struct;
3339
3340                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3341                 if (rc)
3342                         return rc;
3343
3344                 ring->grp_idx = txr->bnapi->index;
3345                 if (bp->tx_push_size) {
3346                         dma_addr_t mapping;
3347
3348                         /* One pre-allocated DMA buffer to backup
3349                          * TX push operation
3350                          */
3351                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
3352                                                 bp->tx_push_size,
3353                                                 &txr->tx_push_mapping,
3354                                                 GFP_KERNEL);
3355
3356                         if (!txr->tx_push)
3357                                 return -ENOMEM;
3358
3359                         mapping = txr->tx_push_mapping +
3360                                 sizeof(struct tx_push_bd);
3361                         txr->data_mapping = cpu_to_le64(mapping);
3362                 }
3363                 qidx = bp->tc_to_qidx[j];
3364                 ring->queue_id = bp->q_info[qidx].queue_id;
3365                 spin_lock_init(&txr->xdp_tx_lock);
3366                 if (i < bp->tx_nr_rings_xdp)
3367                         continue;
3368                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
3369                         j++;
3370         }
3371         return 0;
3372 }
3373
3374 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr)
3375 {
3376         struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3377
3378         kfree(cpr->cp_desc_ring);
3379         cpr->cp_desc_ring = NULL;
3380         ring->ring_mem.pg_arr = NULL;
3381         kfree(cpr->cp_desc_mapping);
3382         cpr->cp_desc_mapping = NULL;
3383         ring->ring_mem.dma_arr = NULL;
3384 }
3385
3386 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n)
3387 {
3388         cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL);
3389         if (!cpr->cp_desc_ring)
3390                 return -ENOMEM;
3391         cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping),
3392                                        GFP_KERNEL);
3393         if (!cpr->cp_desc_mapping)
3394                 return -ENOMEM;
3395         return 0;
3396 }
3397
3398 static void bnxt_free_all_cp_arrays(struct bnxt *bp)
3399 {
3400         int i;
3401
3402         if (!bp->bnapi)
3403                 return;
3404         for (i = 0; i < bp->cp_nr_rings; i++) {
3405                 struct bnxt_napi *bnapi = bp->bnapi[i];
3406
3407                 if (!bnapi)
3408                         continue;
3409                 bnxt_free_cp_arrays(&bnapi->cp_ring);
3410         }
3411 }
3412
3413 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp)
3414 {
3415         int i, n = bp->cp_nr_pages;
3416
3417         for (i = 0; i < bp->cp_nr_rings; i++) {
3418                 struct bnxt_napi *bnapi = bp->bnapi[i];
3419                 int rc;
3420
3421                 if (!bnapi)
3422                         continue;
3423                 rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n);
3424                 if (rc)
3425                         return rc;
3426         }
3427         return 0;
3428 }
3429
3430 static void bnxt_free_cp_rings(struct bnxt *bp)
3431 {
3432         int i;
3433
3434         if (!bp->bnapi)
3435                 return;
3436
3437         for (i = 0; i < bp->cp_nr_rings; i++) {
3438                 struct bnxt_napi *bnapi = bp->bnapi[i];
3439                 struct bnxt_cp_ring_info *cpr;
3440                 struct bnxt_ring_struct *ring;
3441                 int j;
3442
3443                 if (!bnapi)
3444                         continue;
3445
3446                 cpr = &bnapi->cp_ring;
3447                 ring = &cpr->cp_ring_struct;
3448
3449                 bnxt_free_ring(bp, &ring->ring_mem);
3450
3451                 for (j = 0; j < 2; j++) {
3452                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3453
3454                         if (cpr2) {
3455                                 ring = &cpr2->cp_ring_struct;
3456                                 bnxt_free_ring(bp, &ring->ring_mem);
3457                                 bnxt_free_cp_arrays(cpr2);
3458                                 kfree(cpr2);
3459                                 cpr->cp_ring_arr[j] = NULL;
3460                         }
3461                 }
3462         }
3463 }
3464
3465 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3466 {
3467         struct bnxt_ring_mem_info *rmem;
3468         struct bnxt_ring_struct *ring;
3469         struct bnxt_cp_ring_info *cpr;
3470         int rc;
3471
3472         cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3473         if (!cpr)
3474                 return NULL;
3475
3476         rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages);
3477         if (rc) {
3478                 bnxt_free_cp_arrays(cpr);
3479                 kfree(cpr);
3480                 return NULL;
3481         }
3482         ring = &cpr->cp_ring_struct;
3483         rmem = &ring->ring_mem;
3484         rmem->nr_pages = bp->cp_nr_pages;
3485         rmem->page_size = HW_CMPD_RING_SIZE;
3486         rmem->pg_arr = (void **)cpr->cp_desc_ring;
3487         rmem->dma_arr = cpr->cp_desc_mapping;
3488         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3489         rc = bnxt_alloc_ring(bp, rmem);
3490         if (rc) {
3491                 bnxt_free_ring(bp, rmem);
3492                 bnxt_free_cp_arrays(cpr);
3493                 kfree(cpr);
3494                 cpr = NULL;
3495         }
3496         return cpr;
3497 }
3498
3499 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3500 {
3501         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3502         int i, rc, ulp_base_vec, ulp_msix;
3503
3504         ulp_msix = bnxt_get_ulp_msix_num(bp);
3505         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3506         for (i = 0; i < bp->cp_nr_rings; i++) {
3507                 struct bnxt_napi *bnapi = bp->bnapi[i];
3508                 struct bnxt_cp_ring_info *cpr;
3509                 struct bnxt_ring_struct *ring;
3510
3511                 if (!bnapi)
3512                         continue;
3513
3514                 cpr = &bnapi->cp_ring;
3515                 cpr->bnapi = bnapi;
3516                 ring = &cpr->cp_ring_struct;
3517
3518                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3519                 if (rc)
3520                         return rc;
3521
3522                 if (ulp_msix && i >= ulp_base_vec)
3523                         ring->map_idx = i + ulp_msix;
3524                 else
3525                         ring->map_idx = i;
3526
3527                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3528                         continue;
3529
3530                 if (i < bp->rx_nr_rings) {
3531                         struct bnxt_cp_ring_info *cpr2 =
3532                                 bnxt_alloc_cp_sub_ring(bp);
3533
3534                         cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3535                         if (!cpr2)
3536                                 return -ENOMEM;
3537                         cpr2->bnapi = bnapi;
3538                 }
3539                 if ((sh && i < bp->tx_nr_rings) ||
3540                     (!sh && i >= bp->rx_nr_rings)) {
3541                         struct bnxt_cp_ring_info *cpr2 =
3542                                 bnxt_alloc_cp_sub_ring(bp);
3543
3544                         cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3545                         if (!cpr2)
3546                                 return -ENOMEM;
3547                         cpr2->bnapi = bnapi;
3548                 }
3549         }
3550         return 0;
3551 }
3552
3553 static void bnxt_init_ring_struct(struct bnxt *bp)
3554 {
3555         int i;
3556
3557         for (i = 0; i < bp->cp_nr_rings; i++) {
3558                 struct bnxt_napi *bnapi = bp->bnapi[i];
3559                 struct bnxt_ring_mem_info *rmem;
3560                 struct bnxt_cp_ring_info *cpr;
3561                 struct bnxt_rx_ring_info *rxr;
3562                 struct bnxt_tx_ring_info *txr;
3563                 struct bnxt_ring_struct *ring;
3564
3565                 if (!bnapi)
3566                         continue;
3567
3568                 cpr = &bnapi->cp_ring;
3569                 ring = &cpr->cp_ring_struct;
3570                 rmem = &ring->ring_mem;
3571                 rmem->nr_pages = bp->cp_nr_pages;
3572                 rmem->page_size = HW_CMPD_RING_SIZE;
3573                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3574                 rmem->dma_arr = cpr->cp_desc_mapping;
3575                 rmem->vmem_size = 0;
3576
3577                 rxr = bnapi->rx_ring;
3578                 if (!rxr)
3579                         goto skip_rx;
3580
3581                 ring = &rxr->rx_ring_struct;
3582                 rmem = &ring->ring_mem;
3583                 rmem->nr_pages = bp->rx_nr_pages;
3584                 rmem->page_size = HW_RXBD_RING_SIZE;
3585                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3586                 rmem->dma_arr = rxr->rx_desc_mapping;
3587                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3588                 rmem->vmem = (void **)&rxr->rx_buf_ring;
3589
3590                 ring = &rxr->rx_agg_ring_struct;
3591                 rmem = &ring->ring_mem;
3592                 rmem->nr_pages = bp->rx_agg_nr_pages;
3593                 rmem->page_size = HW_RXBD_RING_SIZE;
3594                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3595                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3596                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3597                 rmem->vmem = (void **)&rxr->rx_agg_ring;
3598
3599 skip_rx:
3600                 txr = bnapi->tx_ring;
3601                 if (!txr)
3602                         continue;
3603
3604                 ring = &txr->tx_ring_struct;
3605                 rmem = &ring->ring_mem;
3606                 rmem->nr_pages = bp->tx_nr_pages;
3607                 rmem->page_size = HW_RXBD_RING_SIZE;
3608                 rmem->pg_arr = (void **)txr->tx_desc_ring;
3609                 rmem->dma_arr = txr->tx_desc_mapping;
3610                 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3611                 rmem->vmem = (void **)&txr->tx_buf_ring;
3612         }
3613 }
3614
3615 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3616 {
3617         int i;
3618         u32 prod;
3619         struct rx_bd **rx_buf_ring;
3620
3621         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3622         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3623                 int j;
3624                 struct rx_bd *rxbd;
3625
3626                 rxbd = rx_buf_ring[i];
3627                 if (!rxbd)
3628                         continue;
3629
3630                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3631                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3632                         rxbd->rx_bd_opaque = prod;
3633                 }
3634         }
3635 }
3636
3637 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3638 {
3639         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3640         struct net_device *dev = bp->dev;
3641         u32 prod;
3642         int i;
3643
3644         prod = rxr->rx_prod;
3645         for (i = 0; i < bp->rx_ring_size; i++) {
3646                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3647                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3648                                     ring_nr, i, bp->rx_ring_size);
3649                         break;
3650                 }
3651                 prod = NEXT_RX(prod);
3652         }
3653         rxr->rx_prod = prod;
3654
3655         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3656                 return 0;
3657
3658         prod = rxr->rx_agg_prod;
3659         for (i = 0; i < bp->rx_agg_ring_size; i++) {
3660                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
3661                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3662                                     ring_nr, i, bp->rx_ring_size);
3663                         break;
3664                 }
3665                 prod = NEXT_RX_AGG(prod);
3666         }
3667         rxr->rx_agg_prod = prod;
3668
3669         if (rxr->rx_tpa) {
3670                 dma_addr_t mapping;
3671                 u8 *data;
3672
3673                 for (i = 0; i < bp->max_tpa; i++) {
3674                         data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL);
3675                         if (!data)
3676                                 return -ENOMEM;
3677
3678                         rxr->rx_tpa[i].data = data;
3679                         rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3680                         rxr->rx_tpa[i].mapping = mapping;
3681                 }
3682         }
3683         return 0;
3684 }
3685
3686 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3687 {
3688         struct bnxt_rx_ring_info *rxr;
3689         struct bnxt_ring_struct *ring;
3690         u32 type;
3691
3692         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3693                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3694
3695         if (NET_IP_ALIGN == 2)
3696                 type |= RX_BD_FLAGS_SOP;
3697
3698         rxr = &bp->rx_ring[ring_nr];
3699         ring = &rxr->rx_ring_struct;
3700         bnxt_init_rxbd_pages(ring, type);
3701
3702         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3703                 bpf_prog_add(bp->xdp_prog, 1);
3704                 rxr->xdp_prog = bp->xdp_prog;
3705         }
3706         ring->fw_ring_id = INVALID_HW_RING_ID;
3707
3708         ring = &rxr->rx_agg_ring_struct;
3709         ring->fw_ring_id = INVALID_HW_RING_ID;
3710
3711         if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
3712                 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3713                         RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3714
3715                 bnxt_init_rxbd_pages(ring, type);
3716         }
3717
3718         return bnxt_alloc_one_rx_ring(bp, ring_nr);
3719 }
3720
3721 static void bnxt_init_cp_rings(struct bnxt *bp)
3722 {
3723         int i, j;
3724
3725         for (i = 0; i < bp->cp_nr_rings; i++) {
3726                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3727                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3728
3729                 ring->fw_ring_id = INVALID_HW_RING_ID;
3730                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3731                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3732                 for (j = 0; j < 2; j++) {
3733                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3734
3735                         if (!cpr2)
3736                                 continue;
3737
3738                         ring = &cpr2->cp_ring_struct;
3739                         ring->fw_ring_id = INVALID_HW_RING_ID;
3740                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3741                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3742                 }
3743         }
3744 }
3745
3746 static int bnxt_init_rx_rings(struct bnxt *bp)
3747 {
3748         int i, rc = 0;
3749
3750         if (BNXT_RX_PAGE_MODE(bp)) {
3751                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3752                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3753         } else {
3754                 bp->rx_offset = BNXT_RX_OFFSET;
3755                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3756         }
3757
3758         for (i = 0; i < bp->rx_nr_rings; i++) {
3759                 rc = bnxt_init_one_rx_ring(bp, i);
3760                 if (rc)
3761                         break;
3762         }
3763
3764         return rc;
3765 }
3766
3767 static int bnxt_init_tx_rings(struct bnxt *bp)
3768 {
3769         u16 i;
3770
3771         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3772                                    BNXT_MIN_TX_DESC_CNT);
3773
3774         for (i = 0; i < bp->tx_nr_rings; i++) {
3775                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3776                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3777
3778                 ring->fw_ring_id = INVALID_HW_RING_ID;
3779         }
3780
3781         return 0;
3782 }
3783
3784 static void bnxt_free_ring_grps(struct bnxt *bp)
3785 {
3786         kfree(bp->grp_info);
3787         bp->grp_info = NULL;
3788 }
3789
3790 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3791 {
3792         int i;
3793
3794         if (irq_re_init) {
3795                 bp->grp_info = kcalloc(bp->cp_nr_rings,
3796                                        sizeof(struct bnxt_ring_grp_info),
3797                                        GFP_KERNEL);
3798                 if (!bp->grp_info)
3799                         return -ENOMEM;
3800         }
3801         for (i = 0; i < bp->cp_nr_rings; i++) {
3802                 if (irq_re_init)
3803                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3804                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3805                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3806                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3807                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3808         }
3809         return 0;
3810 }
3811
3812 static void bnxt_free_vnics(struct bnxt *bp)
3813 {
3814         kfree(bp->vnic_info);
3815         bp->vnic_info = NULL;
3816         bp->nr_vnics = 0;
3817 }
3818
3819 static int bnxt_alloc_vnics(struct bnxt *bp)
3820 {
3821         int num_vnics = 1;
3822
3823 #ifdef CONFIG_RFS_ACCEL
3824         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3825                 num_vnics += bp->rx_nr_rings;
3826 #endif
3827
3828         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3829                 num_vnics++;
3830
3831         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3832                                 GFP_KERNEL);
3833         if (!bp->vnic_info)
3834                 return -ENOMEM;
3835
3836         bp->nr_vnics = num_vnics;
3837         return 0;
3838 }
3839
3840 static void bnxt_init_vnics(struct bnxt *bp)
3841 {
3842         int i;
3843
3844         for (i = 0; i < bp->nr_vnics; i++) {
3845                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3846                 int j;
3847
3848                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3849                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3850                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3851
3852                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3853
3854                 if (bp->vnic_info[i].rss_hash_key) {
3855                         if (i == 0)
3856                                 get_random_bytes(vnic->rss_hash_key,
3857                                               HW_HASH_KEY_SIZE);
3858                         else
3859                                 memcpy(vnic->rss_hash_key,
3860                                        bp->vnic_info[0].rss_hash_key,
3861                                        HW_HASH_KEY_SIZE);
3862                 }
3863         }
3864 }
3865
3866 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3867 {
3868         int pages;
3869
3870         pages = ring_size / desc_per_pg;
3871
3872         if (!pages)
3873                 return 1;
3874
3875         pages++;
3876
3877         while (pages & (pages - 1))
3878                 pages++;
3879
3880         return pages;
3881 }
3882
3883 void bnxt_set_tpa_flags(struct bnxt *bp)
3884 {
3885         bp->flags &= ~BNXT_FLAG_TPA;
3886         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3887                 return;
3888         if (bp->dev->features & NETIF_F_LRO)
3889                 bp->flags |= BNXT_FLAG_LRO;
3890         else if (bp->dev->features & NETIF_F_GRO_HW)
3891                 bp->flags |= BNXT_FLAG_GRO;
3892 }
3893
3894 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3895  * be set on entry.
3896  */
3897 void bnxt_set_ring_params(struct bnxt *bp)
3898 {
3899         u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3900         u32 agg_factor = 0, agg_ring_size = 0;
3901
3902         /* 8 for CRC and VLAN */
3903         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3904
3905         rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) +
3906                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3907
3908         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3909         ring_size = bp->rx_ring_size;
3910         bp->rx_agg_ring_size = 0;
3911         bp->rx_agg_nr_pages = 0;
3912
3913         if (bp->flags & BNXT_FLAG_TPA)
3914                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3915
3916         bp->flags &= ~BNXT_FLAG_JUMBO;
3917         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3918                 u32 jumbo_factor;
3919
3920                 bp->flags |= BNXT_FLAG_JUMBO;
3921                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3922                 if (jumbo_factor > agg_factor)
3923                         agg_factor = jumbo_factor;
3924         }
3925         if (agg_factor) {
3926                 if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) {
3927                         ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA;
3928                         netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n",
3929                                     bp->rx_ring_size, ring_size);
3930                         bp->rx_ring_size = ring_size;
3931                 }
3932                 agg_ring_size = ring_size * agg_factor;
3933
3934                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3935                                                         RX_DESC_CNT);
3936                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3937                         u32 tmp = agg_ring_size;
3938
3939                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3940                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3941                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3942                                     tmp, agg_ring_size);
3943                 }
3944                 bp->rx_agg_ring_size = agg_ring_size;
3945                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3946
3947                 if (BNXT_RX_PAGE_MODE(bp)) {
3948                         rx_space = PAGE_SIZE;
3949                         rx_size = PAGE_SIZE -
3950                                   ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) -
3951                                   SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3952                 } else {
3953                         rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3954                         rx_space = rx_size + NET_SKB_PAD +
3955                                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3956                 }
3957         }
3958
3959         bp->rx_buf_use_size = rx_size;
3960         bp->rx_buf_size = rx_space;
3961
3962         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3963         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3964
3965         ring_size = bp->tx_ring_size;
3966         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3967         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3968
3969         max_rx_cmpl = bp->rx_ring_size;
3970         /* MAX TPA needs to be added because TPA_START completions are
3971          * immediately recycled, so the TPA completions are not bound by
3972          * the RX ring size.
3973          */
3974         if (bp->flags & BNXT_FLAG_TPA)
3975                 max_rx_cmpl += bp->max_tpa;
3976         /* RX and TPA completions are 32-byte, all others are 16-byte */
3977         ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3978         bp->cp_ring_size = ring_size;
3979
3980         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3981         if (bp->cp_nr_pages > MAX_CP_PAGES) {
3982                 bp->cp_nr_pages = MAX_CP_PAGES;
3983                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3984                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3985                             ring_size, bp->cp_ring_size);
3986         }
3987         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3988         bp->cp_ring_mask = bp->cp_bit - 1;
3989 }
3990
3991 /* Changing allocation mode of RX rings.
3992  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3993  */
3994 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3995 {
3996         if (page_mode) {
3997                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3998                 bp->flags |= BNXT_FLAG_RX_PAGE_MODE;
3999
4000                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
4001                         bp->flags |= BNXT_FLAG_JUMBO;
4002                         bp->rx_skb_func = bnxt_rx_multi_page_skb;
4003                         bp->dev->max_mtu =
4004                                 min_t(u16, bp->max_mtu, BNXT_MAX_MTU);
4005                 } else {
4006                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
4007                         bp->rx_skb_func = bnxt_rx_page_skb;
4008                         bp->dev->max_mtu =
4009                                 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
4010                 }
4011                 bp->rx_dir = DMA_BIDIRECTIONAL;
4012                 /* Disable LRO or GRO_HW */
4013                 netdev_update_features(bp->dev);
4014         } else {
4015                 bp->dev->max_mtu = bp->max_mtu;
4016                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
4017                 bp->rx_dir = DMA_FROM_DEVICE;
4018                 bp->rx_skb_func = bnxt_rx_skb;
4019         }
4020         return 0;
4021 }
4022
4023 static void bnxt_free_vnic_attributes(struct bnxt *bp)
4024 {
4025         int i;
4026         struct bnxt_vnic_info *vnic;
4027         struct pci_dev *pdev = bp->pdev;
4028
4029         if (!bp->vnic_info)
4030                 return;
4031
4032         for (i = 0; i < bp->nr_vnics; i++) {
4033                 vnic = &bp->vnic_info[i];
4034
4035                 kfree(vnic->fw_grp_ids);
4036                 vnic->fw_grp_ids = NULL;
4037
4038                 kfree(vnic->uc_list);
4039                 vnic->uc_list = NULL;
4040
4041                 if (vnic->mc_list) {
4042                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
4043                                           vnic->mc_list, vnic->mc_list_mapping);
4044                         vnic->mc_list = NULL;
4045                 }
4046
4047                 if (vnic->rss_table) {
4048                         dma_free_coherent(&pdev->dev, vnic->rss_table_size,
4049                                           vnic->rss_table,
4050                                           vnic->rss_table_dma_addr);
4051                         vnic->rss_table = NULL;
4052                 }
4053
4054                 vnic->rss_hash_key = NULL;
4055                 vnic->flags = 0;
4056         }
4057 }
4058
4059 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
4060 {
4061         int i, rc = 0, size;
4062         struct bnxt_vnic_info *vnic;
4063         struct pci_dev *pdev = bp->pdev;
4064         int max_rings;
4065
4066         for (i = 0; i < bp->nr_vnics; i++) {
4067                 vnic = &bp->vnic_info[i];
4068
4069                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
4070                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
4071
4072                         if (mem_size > 0) {
4073                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
4074                                 if (!vnic->uc_list) {
4075                                         rc = -ENOMEM;
4076                                         goto out;
4077                                 }
4078                         }
4079                 }
4080
4081                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
4082                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
4083                         vnic->mc_list =
4084                                 dma_alloc_coherent(&pdev->dev,
4085                                                    vnic->mc_list_size,
4086                                                    &vnic->mc_list_mapping,
4087                                                    GFP_KERNEL);
4088                         if (!vnic->mc_list) {
4089                                 rc = -ENOMEM;
4090                                 goto out;
4091                         }
4092                 }
4093
4094                 if (bp->flags & BNXT_FLAG_CHIP_P5)
4095                         goto vnic_skip_grps;
4096
4097                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4098                         max_rings = bp->rx_nr_rings;
4099                 else
4100                         max_rings = 1;
4101
4102                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
4103                 if (!vnic->fw_grp_ids) {
4104                         rc = -ENOMEM;
4105                         goto out;
4106                 }
4107 vnic_skip_grps:
4108                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
4109                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
4110                         continue;
4111
4112                 /* Allocate rss table and hash key */
4113                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
4114                 if (bp->flags & BNXT_FLAG_CHIP_P5)
4115                         size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
4116
4117                 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
4118                 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
4119                                                      vnic->rss_table_size,
4120                                                      &vnic->rss_table_dma_addr,
4121                                                      GFP_KERNEL);
4122                 if (!vnic->rss_table) {
4123                         rc = -ENOMEM;
4124                         goto out;
4125                 }
4126
4127                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
4128                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
4129         }
4130         return 0;
4131
4132 out:
4133         return rc;
4134 }
4135
4136 static void bnxt_free_hwrm_resources(struct bnxt *bp)
4137 {
4138         struct bnxt_hwrm_wait_token *token;
4139
4140         dma_pool_destroy(bp->hwrm_dma_pool);
4141         bp->hwrm_dma_pool = NULL;
4142
4143         rcu_read_lock();
4144         hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node)
4145                 WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED);
4146         rcu_read_unlock();
4147 }
4148
4149 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
4150 {
4151         bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev,
4152                                             BNXT_HWRM_DMA_SIZE,
4153                                             BNXT_HWRM_DMA_ALIGN, 0);
4154         if (!bp->hwrm_dma_pool)
4155                 return -ENOMEM;
4156
4157         INIT_HLIST_HEAD(&bp->hwrm_pending_list);
4158
4159         return 0;
4160 }
4161
4162 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
4163 {
4164         kfree(stats->hw_masks);
4165         stats->hw_masks = NULL;
4166         kfree(stats->sw_stats);
4167         stats->sw_stats = NULL;
4168         if (stats->hw_stats) {
4169                 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
4170                                   stats->hw_stats_map);
4171                 stats->hw_stats = NULL;
4172         }
4173 }
4174
4175 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
4176                                 bool alloc_masks)
4177 {
4178         stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
4179                                              &stats->hw_stats_map, GFP_KERNEL);
4180         if (!stats->hw_stats)
4181                 return -ENOMEM;
4182
4183         stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
4184         if (!stats->sw_stats)
4185                 goto stats_mem_err;
4186
4187         if (alloc_masks) {
4188                 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
4189                 if (!stats->hw_masks)
4190                         goto stats_mem_err;
4191         }
4192         return 0;
4193
4194 stats_mem_err:
4195         bnxt_free_stats_mem(bp, stats);
4196         return -ENOMEM;
4197 }
4198
4199 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
4200 {
4201         int i;
4202
4203         for (i = 0; i < count; i++)
4204                 mask_arr[i] = mask;
4205 }
4206
4207 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
4208 {
4209         int i;
4210
4211         for (i = 0; i < count; i++)
4212                 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
4213 }
4214
4215 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
4216                                     struct bnxt_stats_mem *stats)
4217 {
4218         struct hwrm_func_qstats_ext_output *resp;
4219         struct hwrm_func_qstats_ext_input *req;
4220         __le64 *hw_masks;
4221         int rc;
4222
4223         if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
4224             !(bp->flags & BNXT_FLAG_CHIP_P5))
4225                 return -EOPNOTSUPP;
4226
4227         rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT);
4228         if (rc)
4229                 return rc;
4230
4231         req->fid = cpu_to_le16(0xffff);
4232         req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4233
4234         resp = hwrm_req_hold(bp, req);
4235         rc = hwrm_req_send(bp, req);
4236         if (!rc) {
4237                 hw_masks = &resp->rx_ucast_pkts;
4238                 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
4239         }
4240         hwrm_req_drop(bp, req);
4241         return rc;
4242 }
4243
4244 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
4245 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
4246
4247 static void bnxt_init_stats(struct bnxt *bp)
4248 {
4249         struct bnxt_napi *bnapi = bp->bnapi[0];
4250         struct bnxt_cp_ring_info *cpr;
4251         struct bnxt_stats_mem *stats;
4252         __le64 *rx_stats, *tx_stats;
4253         int rc, rx_count, tx_count;
4254         u64 *rx_masks, *tx_masks;
4255         u64 mask;
4256         u8 flags;
4257
4258         cpr = &bnapi->cp_ring;
4259         stats = &cpr->stats;
4260         rc = bnxt_hwrm_func_qstat_ext(bp, stats);
4261         if (rc) {
4262                 if (bp->flags & BNXT_FLAG_CHIP_P5)
4263                         mask = (1ULL << 48) - 1;
4264                 else
4265                         mask = -1ULL;
4266                 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
4267         }
4268         if (bp->flags & BNXT_FLAG_PORT_STATS) {
4269                 stats = &bp->port_stats;
4270                 rx_stats = stats->hw_stats;
4271                 rx_masks = stats->hw_masks;
4272                 rx_count = sizeof(struct rx_port_stats) / 8;
4273                 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4274                 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4275                 tx_count = sizeof(struct tx_port_stats) / 8;
4276
4277                 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
4278                 rc = bnxt_hwrm_port_qstats(bp, flags);
4279                 if (rc) {
4280                         mask = (1ULL << 40) - 1;
4281
4282                         bnxt_fill_masks(rx_masks, mask, rx_count);
4283                         bnxt_fill_masks(tx_masks, mask, tx_count);
4284                 } else {
4285                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4286                         bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
4287                         bnxt_hwrm_port_qstats(bp, 0);
4288                 }
4289         }
4290         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
4291                 stats = &bp->rx_port_stats_ext;
4292                 rx_stats = stats->hw_stats;
4293                 rx_masks = stats->hw_masks;
4294                 rx_count = sizeof(struct rx_port_stats_ext) / 8;
4295                 stats = &bp->tx_port_stats_ext;
4296                 tx_stats = stats->hw_stats;
4297                 tx_masks = stats->hw_masks;
4298                 tx_count = sizeof(struct tx_port_stats_ext) / 8;
4299
4300                 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4301                 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
4302                 if (rc) {
4303                         mask = (1ULL << 40) - 1;
4304
4305                         bnxt_fill_masks(rx_masks, mask, rx_count);
4306                         if (tx_stats)
4307                                 bnxt_fill_masks(tx_masks, mask, tx_count);
4308                 } else {
4309                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4310                         if (tx_stats)
4311                                 bnxt_copy_hw_masks(tx_masks, tx_stats,
4312                                                    tx_count);
4313                         bnxt_hwrm_port_qstats_ext(bp, 0);
4314                 }
4315         }
4316 }
4317
4318 static void bnxt_free_port_stats(struct bnxt *bp)
4319 {
4320         bp->flags &= ~BNXT_FLAG_PORT_STATS;
4321         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
4322
4323         bnxt_free_stats_mem(bp, &bp->port_stats);
4324         bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
4325         bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
4326 }
4327
4328 static void bnxt_free_ring_stats(struct bnxt *bp)
4329 {
4330         int i;
4331
4332         if (!bp->bnapi)
4333                 return;
4334
4335         for (i = 0; i < bp->cp_nr_rings; i++) {
4336                 struct bnxt_napi *bnapi = bp->bnapi[i];
4337                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4338
4339                 bnxt_free_stats_mem(bp, &cpr->stats);
4340         }
4341 }
4342
4343 static int bnxt_alloc_stats(struct bnxt *bp)
4344 {
4345         u32 size, i;
4346         int rc;
4347
4348         size = bp->hw_ring_stats_size;
4349
4350         for (i = 0; i < bp->cp_nr_rings; i++) {
4351                 struct bnxt_napi *bnapi = bp->bnapi[i];
4352                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4353
4354                 cpr->stats.len = size;
4355                 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
4356                 if (rc)
4357                         return rc;
4358
4359                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4360         }
4361
4362         if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
4363                 return 0;
4364
4365         if (bp->port_stats.hw_stats)
4366                 goto alloc_ext_stats;
4367
4368         bp->port_stats.len = BNXT_PORT_STATS_SIZE;
4369         rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
4370         if (rc)
4371                 return rc;
4372
4373         bp->flags |= BNXT_FLAG_PORT_STATS;
4374
4375 alloc_ext_stats:
4376         /* Display extended statistics only if FW supports it */
4377         if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
4378                 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
4379                         return 0;
4380
4381         if (bp->rx_port_stats_ext.hw_stats)
4382                 goto alloc_tx_ext_stats;
4383
4384         bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
4385         rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
4386         /* Extended stats are optional */
4387         if (rc)
4388                 return 0;
4389
4390 alloc_tx_ext_stats:
4391         if (bp->tx_port_stats_ext.hw_stats)
4392                 return 0;
4393
4394         if (bp->hwrm_spec_code >= 0x10902 ||
4395             (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
4396                 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
4397                 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
4398                 /* Extended stats are optional */
4399                 if (rc)
4400                         return 0;
4401         }
4402         bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4403         return 0;
4404 }
4405
4406 static void bnxt_clear_ring_indices(struct bnxt *bp)
4407 {
4408         int i;
4409
4410         if (!bp->bnapi)
4411                 return;
4412
4413         for (i = 0; i < bp->cp_nr_rings; i++) {
4414                 struct bnxt_napi *bnapi = bp->bnapi[i];
4415                 struct bnxt_cp_ring_info *cpr;
4416                 struct bnxt_rx_ring_info *rxr;
4417                 struct bnxt_tx_ring_info *txr;
4418
4419                 if (!bnapi)
4420                         continue;
4421
4422                 cpr = &bnapi->cp_ring;
4423                 cpr->cp_raw_cons = 0;
4424
4425                 txr = bnapi->tx_ring;
4426                 if (txr) {
4427                         txr->tx_prod = 0;
4428                         txr->tx_cons = 0;
4429                 }
4430
4431                 rxr = bnapi->rx_ring;
4432                 if (rxr) {
4433                         rxr->rx_prod = 0;
4434                         rxr->rx_agg_prod = 0;
4435                         rxr->rx_sw_agg_prod = 0;
4436                         rxr->rx_next_cons = 0;
4437                 }
4438         }
4439 }
4440
4441 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
4442 {
4443 #ifdef CONFIG_RFS_ACCEL
4444         int i;
4445
4446         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
4447          * safe to delete the hash table.
4448          */
4449         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4450                 struct hlist_head *head;
4451                 struct hlist_node *tmp;
4452                 struct bnxt_ntuple_filter *fltr;
4453
4454                 head = &bp->ntp_fltr_hash_tbl[i];
4455                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4456                         hlist_del(&fltr->hash);
4457                         kfree(fltr);
4458                 }
4459         }
4460         if (irq_reinit) {
4461                 bitmap_free(bp->ntp_fltr_bmap);
4462                 bp->ntp_fltr_bmap = NULL;
4463         }
4464         bp->ntp_fltr_count = 0;
4465 #endif
4466 }
4467
4468 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4469 {
4470 #ifdef CONFIG_RFS_ACCEL
4471         int i, rc = 0;
4472
4473         if (!(bp->flags & BNXT_FLAG_RFS))
4474                 return 0;
4475
4476         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4477                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4478
4479         bp->ntp_fltr_count = 0;
4480         bp->ntp_fltr_bmap = bitmap_zalloc(BNXT_NTP_FLTR_MAX_FLTR, GFP_KERNEL);
4481
4482         if (!bp->ntp_fltr_bmap)
4483                 rc = -ENOMEM;
4484
4485         return rc;
4486 #else
4487         return 0;
4488 #endif
4489 }
4490
4491 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4492 {
4493         bnxt_free_vnic_attributes(bp);
4494         bnxt_free_tx_rings(bp);
4495         bnxt_free_rx_rings(bp);
4496         bnxt_free_cp_rings(bp);
4497         bnxt_free_all_cp_arrays(bp);
4498         bnxt_free_ntp_fltrs(bp, irq_re_init);
4499         if (irq_re_init) {
4500                 bnxt_free_ring_stats(bp);
4501                 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) ||
4502                     test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
4503                         bnxt_free_port_stats(bp);
4504                 bnxt_free_ring_grps(bp);
4505                 bnxt_free_vnics(bp);
4506                 kfree(bp->tx_ring_map);
4507                 bp->tx_ring_map = NULL;
4508                 kfree(bp->tx_ring);
4509                 bp->tx_ring = NULL;
4510                 kfree(bp->rx_ring);
4511                 bp->rx_ring = NULL;
4512                 kfree(bp->bnapi);
4513                 bp->bnapi = NULL;
4514         } else {
4515                 bnxt_clear_ring_indices(bp);
4516         }
4517 }
4518
4519 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4520 {
4521         int i, j, rc, size, arr_size;
4522         void *bnapi;
4523
4524         if (irq_re_init) {
4525                 /* Allocate bnapi mem pointer array and mem block for
4526                  * all queues
4527                  */
4528                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4529                                 bp->cp_nr_rings);
4530                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4531                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4532                 if (!bnapi)
4533                         return -ENOMEM;
4534
4535                 bp->bnapi = bnapi;
4536                 bnapi += arr_size;
4537                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4538                         bp->bnapi[i] = bnapi;
4539                         bp->bnapi[i]->index = i;
4540                         bp->bnapi[i]->bp = bp;
4541                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4542                                 struct bnxt_cp_ring_info *cpr =
4543                                         &bp->bnapi[i]->cp_ring;
4544
4545                                 cpr->cp_ring_struct.ring_mem.flags =
4546                                         BNXT_RMEM_RING_PTE_FLAG;
4547                         }
4548                 }
4549
4550                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
4551                                       sizeof(struct bnxt_rx_ring_info),
4552                                       GFP_KERNEL);
4553                 if (!bp->rx_ring)
4554                         return -ENOMEM;
4555
4556                 for (i = 0; i < bp->rx_nr_rings; i++) {
4557                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4558
4559                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4560                                 rxr->rx_ring_struct.ring_mem.flags =
4561                                         BNXT_RMEM_RING_PTE_FLAG;
4562                                 rxr->rx_agg_ring_struct.ring_mem.flags =
4563                                         BNXT_RMEM_RING_PTE_FLAG;
4564                         }
4565                         rxr->bnapi = bp->bnapi[i];
4566                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4567                 }
4568
4569                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4570                                       sizeof(struct bnxt_tx_ring_info),
4571                                       GFP_KERNEL);
4572                 if (!bp->tx_ring)
4573                         return -ENOMEM;
4574
4575                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4576                                           GFP_KERNEL);
4577
4578                 if (!bp->tx_ring_map)
4579                         return -ENOMEM;
4580
4581                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4582                         j = 0;
4583                 else
4584                         j = bp->rx_nr_rings;
4585
4586                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4587                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4588
4589                         if (bp->flags & BNXT_FLAG_CHIP_P5)
4590                                 txr->tx_ring_struct.ring_mem.flags =
4591                                         BNXT_RMEM_RING_PTE_FLAG;
4592                         txr->bnapi = bp->bnapi[j];
4593                         bp->bnapi[j]->tx_ring = txr;
4594                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4595                         if (i >= bp->tx_nr_rings_xdp) {
4596                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
4597                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
4598                         } else {
4599                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4600                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4601                         }
4602                 }
4603
4604                 rc = bnxt_alloc_stats(bp);
4605                 if (rc)
4606                         goto alloc_mem_err;
4607                 bnxt_init_stats(bp);
4608
4609                 rc = bnxt_alloc_ntp_fltrs(bp);
4610                 if (rc)
4611                         goto alloc_mem_err;
4612
4613                 rc = bnxt_alloc_vnics(bp);
4614                 if (rc)
4615                         goto alloc_mem_err;
4616         }
4617
4618         rc = bnxt_alloc_all_cp_arrays(bp);
4619         if (rc)
4620                 goto alloc_mem_err;
4621
4622         bnxt_init_ring_struct(bp);
4623
4624         rc = bnxt_alloc_rx_rings(bp);
4625         if (rc)
4626                 goto alloc_mem_err;
4627
4628         rc = bnxt_alloc_tx_rings(bp);
4629         if (rc)
4630                 goto alloc_mem_err;
4631
4632         rc = bnxt_alloc_cp_rings(bp);
4633         if (rc)
4634                 goto alloc_mem_err;
4635
4636         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4637                                   BNXT_VNIC_UCAST_FLAG;
4638         rc = bnxt_alloc_vnic_attributes(bp);
4639         if (rc)
4640                 goto alloc_mem_err;
4641         return 0;
4642
4643 alloc_mem_err:
4644         bnxt_free_mem(bp, true);
4645         return rc;
4646 }
4647
4648 static void bnxt_disable_int(struct bnxt *bp)
4649 {
4650         int i;
4651
4652         if (!bp->bnapi)
4653                 return;
4654
4655         for (i = 0; i < bp->cp_nr_rings; i++) {
4656                 struct bnxt_napi *bnapi = bp->bnapi[i];
4657                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4658                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4659
4660                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
4661                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4662         }
4663 }
4664
4665 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4666 {
4667         struct bnxt_napi *bnapi = bp->bnapi[n];
4668         struct bnxt_cp_ring_info *cpr;
4669
4670         cpr = &bnapi->cp_ring;
4671         return cpr->cp_ring_struct.map_idx;
4672 }
4673
4674 static void bnxt_disable_int_sync(struct bnxt *bp)
4675 {
4676         int i;
4677
4678         if (!bp->irq_tbl)
4679                 return;
4680
4681         atomic_inc(&bp->intr_sem);
4682
4683         bnxt_disable_int(bp);
4684         for (i = 0; i < bp->cp_nr_rings; i++) {
4685                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4686
4687                 synchronize_irq(bp->irq_tbl[map_idx].vector);
4688         }
4689 }
4690
4691 static void bnxt_enable_int(struct bnxt *bp)
4692 {
4693         int i;
4694
4695         atomic_set(&bp->intr_sem, 0);
4696         for (i = 0; i < bp->cp_nr_rings; i++) {
4697                 struct bnxt_napi *bnapi = bp->bnapi[i];
4698                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4699
4700                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4701         }
4702 }
4703
4704 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4705                             bool async_only)
4706 {
4707         DECLARE_BITMAP(async_events_bmap, 256);
4708         u32 *events = (u32 *)async_events_bmap;
4709         struct hwrm_func_drv_rgtr_output *resp;
4710         struct hwrm_func_drv_rgtr_input *req;
4711         u32 flags;
4712         int rc, i;
4713
4714         rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR);
4715         if (rc)
4716                 return rc;
4717
4718         req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4719                                    FUNC_DRV_RGTR_REQ_ENABLES_VER |
4720                                    FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4721
4722         req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4723         flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4724         if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4725                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4726         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4727                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4728                          FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4729         req->flags = cpu_to_le32(flags);
4730         req->ver_maj_8b = DRV_VER_MAJ;
4731         req->ver_min_8b = DRV_VER_MIN;
4732         req->ver_upd_8b = DRV_VER_UPD;
4733         req->ver_maj = cpu_to_le16(DRV_VER_MAJ);
4734         req->ver_min = cpu_to_le16(DRV_VER_MIN);
4735         req->ver_upd = cpu_to_le16(DRV_VER_UPD);
4736
4737         if (BNXT_PF(bp)) {
4738                 u32 data[8];
4739                 int i;
4740
4741                 memset(data, 0, sizeof(data));
4742                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4743                         u16 cmd = bnxt_vf_req_snif[i];
4744                         unsigned int bit, idx;
4745
4746                         idx = cmd / 32;
4747                         bit = cmd % 32;
4748                         data[idx] |= 1 << bit;
4749                 }
4750
4751                 for (i = 0; i < 8; i++)
4752                         req->vf_req_fwd[i] = cpu_to_le32(data[i]);
4753
4754                 req->enables |=
4755                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4756         }
4757
4758         if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4759                 req->flags |= cpu_to_le32(
4760                         FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4761
4762         memset(async_events_bmap, 0, sizeof(async_events_bmap));
4763         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4764                 u16 event_id = bnxt_async_events_arr[i];
4765
4766                 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4767                     !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4768                         continue;
4769                 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE &&
4770                     !bp->ptp_cfg)
4771                         continue;
4772                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4773         }
4774         if (bmap && bmap_size) {
4775                 for (i = 0; i < bmap_size; i++) {
4776                         if (test_bit(i, bmap))
4777                                 __set_bit(i, async_events_bmap);
4778                 }
4779         }
4780         for (i = 0; i < 8; i++)
4781                 req->async_event_fwd[i] |= cpu_to_le32(events[i]);
4782
4783         if (async_only)
4784                 req->enables =
4785                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4786
4787         resp = hwrm_req_hold(bp, req);
4788         rc = hwrm_req_send(bp, req);
4789         if (!rc) {
4790                 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4791                 if (resp->flags &
4792                     cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4793                         bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4794         }
4795         hwrm_req_drop(bp, req);
4796         return rc;
4797 }
4798
4799 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4800 {
4801         struct hwrm_func_drv_unrgtr_input *req;
4802         int rc;
4803
4804         if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4805                 return 0;
4806
4807         rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR);
4808         if (rc)
4809                 return rc;
4810         return hwrm_req_send(bp, req);
4811 }
4812
4813 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4814 {
4815         struct hwrm_tunnel_dst_port_free_input *req;
4816         int rc;
4817
4818         if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN &&
4819             bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID)
4820                 return 0;
4821         if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE &&
4822             bp->nge_fw_dst_port_id == INVALID_HW_RING_ID)
4823                 return 0;
4824
4825         rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE);
4826         if (rc)
4827                 return rc;
4828
4829         req->tunnel_type = tunnel_type;
4830
4831         switch (tunnel_type) {
4832         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4833                 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4834                 bp->vxlan_port = 0;
4835                 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4836                 break;
4837         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4838                 req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4839                 bp->nge_port = 0;
4840                 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4841                 break;
4842         default:
4843                 break;
4844         }
4845
4846         rc = hwrm_req_send(bp, req);
4847         if (rc)
4848                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4849                            rc);
4850         return rc;
4851 }
4852
4853 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4854                                            u8 tunnel_type)
4855 {
4856         struct hwrm_tunnel_dst_port_alloc_output *resp;
4857         struct hwrm_tunnel_dst_port_alloc_input *req;
4858         int rc;
4859
4860         rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC);
4861         if (rc)
4862                 return rc;
4863
4864         req->tunnel_type = tunnel_type;
4865         req->tunnel_dst_port_val = port;
4866
4867         resp = hwrm_req_hold(bp, req);
4868         rc = hwrm_req_send(bp, req);
4869         if (rc) {
4870                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4871                            rc);
4872                 goto err_out;
4873         }
4874
4875         switch (tunnel_type) {
4876         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4877                 bp->vxlan_port = port;
4878                 bp->vxlan_fw_dst_port_id =
4879                         le16_to_cpu(resp->tunnel_dst_port_id);
4880                 break;
4881         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4882                 bp->nge_port = port;
4883                 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4884                 break;
4885         default:
4886                 break;
4887         }
4888
4889 err_out:
4890         hwrm_req_drop(bp, req);
4891         return rc;
4892 }
4893
4894 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4895 {
4896         struct hwrm_cfa_l2_set_rx_mask_input *req;
4897         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4898         int rc;
4899
4900         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK);
4901         if (rc)
4902                 return rc;
4903
4904         req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4905         if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) {
4906                 req->num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4907                 req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4908         }
4909         req->mask = cpu_to_le32(vnic->rx_mask);
4910         return hwrm_req_send_silent(bp, req);
4911 }
4912
4913 #ifdef CONFIG_RFS_ACCEL
4914 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4915                                             struct bnxt_ntuple_filter *fltr)
4916 {
4917         struct hwrm_cfa_ntuple_filter_free_input *req;
4918         int rc;
4919
4920         rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE);
4921         if (rc)
4922                 return rc;
4923
4924         req->ntuple_filter_id = fltr->filter_id;
4925         return hwrm_req_send(bp, req);
4926 }
4927
4928 #define BNXT_NTP_FLTR_FLAGS                                     \
4929         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
4930          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
4931          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
4932          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
4933          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
4934          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
4935          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
4936          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
4937          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
4938          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
4939          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
4940          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
4941          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
4942          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4943
4944 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
4945                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4946
4947 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4948                                              struct bnxt_ntuple_filter *fltr)
4949 {
4950         struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4951         struct hwrm_cfa_ntuple_filter_alloc_input *req;
4952         struct flow_keys *keys = &fltr->fkeys;
4953         struct bnxt_vnic_info *vnic;
4954         u32 flags = 0;
4955         int rc;
4956
4957         rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC);
4958         if (rc)
4959                 return rc;
4960
4961         req->l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4962
4963         if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4964                 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4965                 req->dst_id = cpu_to_le16(fltr->rxq);
4966         } else {
4967                 vnic = &bp->vnic_info[fltr->rxq + 1];
4968                 req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
4969         }
4970         req->flags = cpu_to_le32(flags);
4971         req->enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4972
4973         req->ethertype = htons(ETH_P_IP);
4974         memcpy(req->src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4975         req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4976         req->ip_protocol = keys->basic.ip_proto;
4977
4978         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4979                 int i;
4980
4981                 req->ethertype = htons(ETH_P_IPV6);
4982                 req->ip_addr_type =
4983                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4984                 *(struct in6_addr *)&req->src_ipaddr[0] =
4985                         keys->addrs.v6addrs.src;
4986                 *(struct in6_addr *)&req->dst_ipaddr[0] =
4987                         keys->addrs.v6addrs.dst;
4988                 for (i = 0; i < 4; i++) {
4989                         req->src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4990                         req->dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4991                 }
4992         } else {
4993                 req->src_ipaddr[0] = keys->addrs.v4addrs.src;
4994                 req->src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4995                 req->dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4996                 req->dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4997         }
4998         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4999                 req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
5000                 req->tunnel_type =
5001                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
5002         }
5003
5004         req->src_port = keys->ports.src;
5005         req->src_port_mask = cpu_to_be16(0xffff);
5006         req->dst_port = keys->ports.dst;
5007         req->dst_port_mask = cpu_to_be16(0xffff);
5008
5009         resp = hwrm_req_hold(bp, req);
5010         rc = hwrm_req_send(bp, req);
5011         if (!rc)
5012                 fltr->filter_id = resp->ntuple_filter_id;
5013         hwrm_req_drop(bp, req);
5014         return rc;
5015 }
5016 #endif
5017
5018 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
5019                                      const u8 *mac_addr)
5020 {
5021         struct hwrm_cfa_l2_filter_alloc_output *resp;
5022         struct hwrm_cfa_l2_filter_alloc_input *req;
5023         int rc;
5024
5025         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC);
5026         if (rc)
5027                 return rc;
5028
5029         req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
5030         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
5031                 req->flags |=
5032                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
5033         req->dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
5034         req->enables =
5035                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
5036                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
5037                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
5038         memcpy(req->l2_addr, mac_addr, ETH_ALEN);
5039         req->l2_addr_mask[0] = 0xff;
5040         req->l2_addr_mask[1] = 0xff;
5041         req->l2_addr_mask[2] = 0xff;
5042         req->l2_addr_mask[3] = 0xff;
5043         req->l2_addr_mask[4] = 0xff;
5044         req->l2_addr_mask[5] = 0xff;
5045
5046         resp = hwrm_req_hold(bp, req);
5047         rc = hwrm_req_send(bp, req);
5048         if (!rc)
5049                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
5050                                                         resp->l2_filter_id;
5051         hwrm_req_drop(bp, req);
5052         return rc;
5053 }
5054
5055 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
5056 {
5057         struct hwrm_cfa_l2_filter_free_input *req;
5058         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
5059         int rc;
5060
5061         /* Any associated ntuple filters will also be cleared by firmware. */
5062         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
5063         if (rc)
5064                 return rc;
5065         hwrm_req_hold(bp, req);
5066         for (i = 0; i < num_of_vnics; i++) {
5067                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5068
5069                 for (j = 0; j < vnic->uc_filter_count; j++) {
5070                         req->l2_filter_id = vnic->fw_l2_filter_id[j];
5071
5072                         rc = hwrm_req_send(bp, req);
5073                 }
5074                 vnic->uc_filter_count = 0;
5075         }
5076         hwrm_req_drop(bp, req);
5077         return rc;
5078 }
5079
5080 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
5081 {
5082         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5083         u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
5084         struct hwrm_vnic_tpa_cfg_input *req;
5085         int rc;
5086
5087         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
5088                 return 0;
5089
5090         rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG);
5091         if (rc)
5092                 return rc;
5093
5094         if (tpa_flags) {
5095                 u16 mss = bp->dev->mtu - 40;
5096                 u32 nsegs, n, segs = 0, flags;
5097
5098                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
5099                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
5100                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
5101                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
5102                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
5103                 if (tpa_flags & BNXT_FLAG_GRO)
5104                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
5105
5106                 req->flags = cpu_to_le32(flags);
5107
5108                 req->enables =
5109                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
5110                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
5111                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
5112
5113                 /* Number of segs are log2 units, and first packet is not
5114                  * included as part of this units.
5115                  */
5116                 if (mss <= BNXT_RX_PAGE_SIZE) {
5117                         n = BNXT_RX_PAGE_SIZE / mss;
5118                         nsegs = (MAX_SKB_FRAGS - 1) * n;
5119                 } else {
5120                         n = mss / BNXT_RX_PAGE_SIZE;
5121                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
5122                                 n++;
5123                         nsegs = (MAX_SKB_FRAGS - n) / n;
5124                 }
5125
5126                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5127                         segs = MAX_TPA_SEGS_P5;
5128                         max_aggs = bp->max_tpa;
5129                 } else {
5130                         segs = ilog2(nsegs);
5131                 }
5132                 req->max_agg_segs = cpu_to_le16(segs);
5133                 req->max_aggs = cpu_to_le16(max_aggs);
5134
5135                 req->min_agg_len = cpu_to_le32(512);
5136         }
5137         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5138
5139         return hwrm_req_send(bp, req);
5140 }
5141
5142 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
5143 {
5144         struct bnxt_ring_grp_info *grp_info;
5145
5146         grp_info = &bp->grp_info[ring->grp_idx];
5147         return grp_info->cp_fw_ring_id;
5148 }
5149
5150 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
5151 {
5152         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5153                 struct bnxt_napi *bnapi = rxr->bnapi;
5154                 struct bnxt_cp_ring_info *cpr;
5155
5156                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
5157                 return cpr->cp_ring_struct.fw_ring_id;
5158         } else {
5159                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
5160         }
5161 }
5162
5163 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
5164 {
5165         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5166                 struct bnxt_napi *bnapi = txr->bnapi;
5167                 struct bnxt_cp_ring_info *cpr;
5168
5169                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
5170                 return cpr->cp_ring_struct.fw_ring_id;
5171         } else {
5172                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
5173         }
5174 }
5175
5176 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
5177 {
5178         int entries;
5179
5180         if (bp->flags & BNXT_FLAG_CHIP_P5)
5181                 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
5182         else
5183                 entries = HW_HASH_INDEX_SIZE;
5184
5185         bp->rss_indir_tbl_entries = entries;
5186         bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
5187                                           GFP_KERNEL);
5188         if (!bp->rss_indir_tbl)
5189                 return -ENOMEM;
5190         return 0;
5191 }
5192
5193 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
5194 {
5195         u16 max_rings, max_entries, pad, i;
5196
5197         if (!bp->rx_nr_rings)
5198                 return;
5199
5200         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5201                 max_rings = bp->rx_nr_rings - 1;
5202         else
5203                 max_rings = bp->rx_nr_rings;
5204
5205         max_entries = bnxt_get_rxfh_indir_size(bp->dev);
5206
5207         for (i = 0; i < max_entries; i++)
5208                 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
5209
5210         pad = bp->rss_indir_tbl_entries - max_entries;
5211         if (pad)
5212                 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
5213 }
5214
5215 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
5216 {
5217         u16 i, tbl_size, max_ring = 0;
5218
5219         if (!bp->rss_indir_tbl)
5220                 return 0;
5221
5222         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5223         for (i = 0; i < tbl_size; i++)
5224                 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5225         return max_ring;
5226 }
5227
5228 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5229 {
5230         if (bp->flags & BNXT_FLAG_CHIP_P5)
5231                 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5232         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5233                 return 2;
5234         return 1;
5235 }
5236
5237 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5238 {
5239         bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5240         u16 i, j;
5241
5242         /* Fill the RSS indirection table with ring group ids */
5243         for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5244                 if (!no_rss)
5245                         j = bp->rss_indir_tbl[i];
5246                 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5247         }
5248 }
5249
5250 static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5251                                     struct bnxt_vnic_info *vnic)
5252 {
5253         __le16 *ring_tbl = vnic->rss_table;
5254         struct bnxt_rx_ring_info *rxr;
5255         u16 tbl_size, i;
5256
5257         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5258
5259         for (i = 0; i < tbl_size; i++) {
5260                 u16 ring_id, j;
5261
5262                 j = bp->rss_indir_tbl[i];
5263                 rxr = &bp->rx_ring[j];
5264
5265                 ring_id = rxr->rx_ring_struct.fw_ring_id;
5266                 *ring_tbl++ = cpu_to_le16(ring_id);
5267                 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5268                 *ring_tbl++ = cpu_to_le16(ring_id);
5269         }
5270 }
5271
5272 static void
5273 __bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req,
5274                          struct bnxt_vnic_info *vnic)
5275 {
5276         if (bp->flags & BNXT_FLAG_CHIP_P5)
5277                 bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5278         else
5279                 bnxt_fill_hw_rss_tbl(bp, vnic);
5280
5281         if (bp->rss_hash_delta) {
5282                 req->hash_type = cpu_to_le32(bp->rss_hash_delta);
5283                 if (bp->rss_hash_cfg & bp->rss_hash_delta)
5284                         req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE;
5285                 else
5286                         req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE;
5287         } else {
5288                 req->hash_type = cpu_to_le32(bp->rss_hash_cfg);
5289         }
5290         req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5291         req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5292         req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5293 }
5294
5295 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5296 {
5297         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5298         struct hwrm_vnic_rss_cfg_input *req;
5299         int rc;
5300
5301         if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5302             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5303                 return 0;
5304
5305         rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
5306         if (rc)
5307                 return rc;
5308
5309         if (set_rss)
5310                 __bnxt_hwrm_vnic_set_rss(bp, req, vnic);
5311         req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5312         return hwrm_req_send(bp, req);
5313 }
5314
5315 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5316 {
5317         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5318         struct hwrm_vnic_rss_cfg_input *req;
5319         dma_addr_t ring_tbl_map;
5320         u32 i, nr_ctxs;
5321         int rc;
5322
5323         rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
5324         if (rc)
5325                 return rc;
5326
5327         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5328         if (!set_rss)
5329                 return hwrm_req_send(bp, req);
5330
5331         __bnxt_hwrm_vnic_set_rss(bp, req, vnic);
5332         ring_tbl_map = vnic->rss_table_dma_addr;
5333         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5334
5335         hwrm_req_hold(bp, req);
5336         for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5337                 req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5338                 req->ring_table_pair_index = i;
5339                 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5340                 rc = hwrm_req_send(bp, req);
5341                 if (rc)
5342                         goto exit;
5343         }
5344
5345 exit:
5346         hwrm_req_drop(bp, req);
5347         return rc;
5348 }
5349
5350 static void bnxt_hwrm_update_rss_hash_cfg(struct bnxt *bp)
5351 {
5352         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5353         struct hwrm_vnic_rss_qcfg_output *resp;
5354         struct hwrm_vnic_rss_qcfg_input *req;
5355
5356         if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_QCFG))
5357                 return;
5358
5359         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5360         /* all contexts configured to same hash_type, zero always exists */
5361         req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5362         resp = hwrm_req_hold(bp, req);
5363         if (!hwrm_req_send(bp, req)) {
5364                 bp->rss_hash_cfg = le32_to_cpu(resp->hash_type) ?: bp->rss_hash_cfg;
5365                 bp->rss_hash_delta = 0;
5366         }
5367         hwrm_req_drop(bp, req);
5368 }
5369
5370 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5371 {
5372         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5373         struct hwrm_vnic_plcmodes_cfg_input *req;
5374         int rc;
5375
5376         rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG);
5377         if (rc)
5378                 return rc;
5379
5380         req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT);
5381         req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID);
5382
5383         if (BNXT_RX_PAGE_MODE(bp)) {
5384                 req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size);
5385         } else {
5386                 req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5387                                           VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5388                 req->enables |=
5389                         cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5390                 req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5391                 req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5392         }
5393         req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5394         return hwrm_req_send(bp, req);
5395 }
5396
5397 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5398                                         u16 ctx_idx)
5399 {
5400         struct hwrm_vnic_rss_cos_lb_ctx_free_input *req;
5401
5402         if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE))
5403                 return;
5404
5405         req->rss_cos_lb_ctx_id =
5406                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5407
5408         hwrm_req_send(bp, req);
5409         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5410 }
5411
5412 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5413 {
5414         int i, j;
5415
5416         for (i = 0; i < bp->nr_vnics; i++) {
5417                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5418
5419                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5420                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5421                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5422                 }
5423         }
5424         bp->rsscos_nr_ctxs = 0;
5425 }
5426
5427 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5428 {
5429         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp;
5430         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req;
5431         int rc;
5432
5433         rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC);
5434         if (rc)
5435                 return rc;
5436
5437         resp = hwrm_req_hold(bp, req);
5438         rc = hwrm_req_send(bp, req);
5439         if (!rc)
5440                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5441                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
5442         hwrm_req_drop(bp, req);
5443
5444         return rc;
5445 }
5446
5447 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5448 {
5449         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5450                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5451         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5452 }
5453
5454 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5455 {
5456         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5457         struct hwrm_vnic_cfg_input *req;
5458         unsigned int ring = 0, grp_idx;
5459         u16 def_vlan = 0;
5460         int rc;
5461
5462         rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG);
5463         if (rc)
5464                 return rc;
5465
5466         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5467                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5468
5469                 req->default_rx_ring_id =
5470                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5471                 req->default_cmpl_ring_id =
5472                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5473                 req->enables =
5474                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5475                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5476                 goto vnic_mru;
5477         }
5478         req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5479         /* Only RSS support for now TBD: COS & LB */
5480         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5481                 req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5482                 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5483                                            VNIC_CFG_REQ_ENABLES_MRU);
5484         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5485                 req->rss_rule =
5486                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5487                 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5488                                            VNIC_CFG_REQ_ENABLES_MRU);
5489                 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5490         } else {
5491                 req->rss_rule = cpu_to_le16(0xffff);
5492         }
5493
5494         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5495             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5496                 req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5497                 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5498         } else {
5499                 req->cos_rule = cpu_to_le16(0xffff);
5500         }
5501
5502         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5503                 ring = 0;
5504         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5505                 ring = vnic_id - 1;
5506         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5507                 ring = bp->rx_nr_rings - 1;
5508
5509         grp_idx = bp->rx_ring[ring].bnapi->index;
5510         req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5511         req->lb_rule = cpu_to_le16(0xffff);
5512 vnic_mru:
5513         req->mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5514
5515         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5516 #ifdef CONFIG_BNXT_SRIOV
5517         if (BNXT_VF(bp))
5518                 def_vlan = bp->vf.vlan;
5519 #endif
5520         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5521                 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5522         if (!vnic_id && bnxt_ulp_registered(bp->edev))
5523                 req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5524
5525         return hwrm_req_send(bp, req);
5526 }
5527
5528 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5529 {
5530         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5531                 struct hwrm_vnic_free_input *req;
5532
5533                 if (hwrm_req_init(bp, req, HWRM_VNIC_FREE))
5534                         return;
5535
5536                 req->vnic_id =
5537                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5538
5539                 hwrm_req_send(bp, req);
5540                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5541         }
5542 }
5543
5544 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5545 {
5546         u16 i;
5547
5548         for (i = 0; i < bp->nr_vnics; i++)
5549                 bnxt_hwrm_vnic_free_one(bp, i);
5550 }
5551
5552 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5553                                 unsigned int start_rx_ring_idx,
5554                                 unsigned int nr_rings)
5555 {
5556         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5557         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5558         struct hwrm_vnic_alloc_output *resp;
5559         struct hwrm_vnic_alloc_input *req;
5560         int rc;
5561
5562         rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC);
5563         if (rc)
5564                 return rc;
5565
5566         if (bp->flags & BNXT_FLAG_CHIP_P5)
5567                 goto vnic_no_ring_grps;
5568
5569         /* map ring groups to this vnic */
5570         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5571                 grp_idx = bp->rx_ring[i].bnapi->index;
5572                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5573                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5574                                    j, nr_rings);
5575                         break;
5576                 }
5577                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5578         }
5579
5580 vnic_no_ring_grps:
5581         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5582                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5583         if (vnic_id == 0)
5584                 req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5585
5586         resp = hwrm_req_hold(bp, req);
5587         rc = hwrm_req_send(bp, req);
5588         if (!rc)
5589                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5590         hwrm_req_drop(bp, req);
5591         return rc;
5592 }
5593
5594 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5595 {
5596         struct hwrm_vnic_qcaps_output *resp;
5597         struct hwrm_vnic_qcaps_input *req;
5598         int rc;
5599
5600         bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5601         bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5602         if (bp->hwrm_spec_code < 0x10600)
5603                 return 0;
5604
5605         rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS);
5606         if (rc)
5607                 return rc;
5608
5609         resp = hwrm_req_hold(bp, req);
5610         rc = hwrm_req_send(bp, req);
5611         if (!rc) {
5612                 u32 flags = le32_to_cpu(resp->flags);
5613
5614                 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5615                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5616                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5617                 if (flags &
5618                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5619                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5620
5621                 /* Older P5 fw before EXT_HW_STATS support did not set
5622                  * VLAN_STRIP_CAP properly.
5623                  */
5624                 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5625                     (BNXT_CHIP_P5_THOR(bp) &&
5626                      !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5627                         bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5628                 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP)
5629                         bp->fw_cap |= BNXT_FW_CAP_RSS_HASH_TYPE_DELTA;
5630                 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5631                 if (bp->max_tpa_v2) {
5632                         if (BNXT_CHIP_P5_THOR(bp))
5633                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
5634                         else
5635                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2;
5636                 }
5637         }
5638         hwrm_req_drop(bp, req);
5639         return rc;
5640 }
5641
5642 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5643 {
5644         struct hwrm_ring_grp_alloc_output *resp;
5645         struct hwrm_ring_grp_alloc_input *req;
5646         int rc;
5647         u16 i;
5648
5649         if (bp->flags & BNXT_FLAG_CHIP_P5)
5650                 return 0;
5651
5652         rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC);
5653         if (rc)
5654                 return rc;
5655
5656         resp = hwrm_req_hold(bp, req);
5657         for (i = 0; i < bp->rx_nr_rings; i++) {
5658                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5659
5660                 req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5661                 req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5662                 req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5663                 req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5664
5665                 rc = hwrm_req_send(bp, req);
5666
5667                 if (rc)
5668                         break;
5669
5670                 bp->grp_info[grp_idx].fw_grp_id =
5671                         le32_to_cpu(resp->ring_group_id);
5672         }
5673         hwrm_req_drop(bp, req);
5674         return rc;
5675 }
5676
5677 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5678 {
5679         struct hwrm_ring_grp_free_input *req;
5680         u16 i;
5681
5682         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5683                 return;
5684
5685         if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE))
5686                 return;
5687
5688         hwrm_req_hold(bp, req);
5689         for (i = 0; i < bp->cp_nr_rings; i++) {
5690                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5691                         continue;
5692                 req->ring_group_id =
5693                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
5694
5695                 hwrm_req_send(bp, req);
5696                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5697         }
5698         hwrm_req_drop(bp, req);
5699 }
5700
5701 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5702                                     struct bnxt_ring_struct *ring,
5703                                     u32 ring_type, u32 map_index)
5704 {
5705         struct hwrm_ring_alloc_output *resp;
5706         struct hwrm_ring_alloc_input *req;
5707         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5708         struct bnxt_ring_grp_info *grp_info;
5709         int rc, err = 0;
5710         u16 ring_id;
5711
5712         rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC);
5713         if (rc)
5714                 goto exit;
5715
5716         req->enables = 0;
5717         if (rmem->nr_pages > 1) {
5718                 req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5719                 /* Page size is in log2 units */
5720                 req->page_size = BNXT_PAGE_SHIFT;
5721                 req->page_tbl_depth = 1;
5722         } else {
5723                 req->page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5724         }
5725         req->fbo = 0;
5726         /* Association of ring index with doorbell index and MSIX number */
5727         req->logical_id = cpu_to_le16(map_index);
5728
5729         switch (ring_type) {
5730         case HWRM_RING_ALLOC_TX: {
5731                 struct bnxt_tx_ring_info *txr;
5732
5733                 txr = container_of(ring, struct bnxt_tx_ring_info,
5734                                    tx_ring_struct);
5735                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5736                 /* Association of transmit ring with completion ring */
5737                 grp_info = &bp->grp_info[ring->grp_idx];
5738                 req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5739                 req->length = cpu_to_le32(bp->tx_ring_mask + 1);
5740                 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5741                 req->queue_id = cpu_to_le16(ring->queue_id);
5742                 break;
5743         }
5744         case HWRM_RING_ALLOC_RX:
5745                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5746                 req->length = cpu_to_le32(bp->rx_ring_mask + 1);
5747                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5748                         u16 flags = 0;
5749
5750                         /* Association of rx ring with stats context */
5751                         grp_info = &bp->grp_info[ring->grp_idx];
5752                         req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5753                         req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5754                         req->enables |= cpu_to_le32(
5755                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5756                         if (NET_IP_ALIGN == 2)
5757                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5758                         req->flags = cpu_to_le16(flags);
5759                 }
5760                 break;
5761         case HWRM_RING_ALLOC_AGG:
5762                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5763                         req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5764                         /* Association of agg ring with rx ring */
5765                         grp_info = &bp->grp_info[ring->grp_idx];
5766                         req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5767                         req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5768                         req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5769                         req->enables |= cpu_to_le32(
5770                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5771                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5772                 } else {
5773                         req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5774                 }
5775                 req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5776                 break;
5777         case HWRM_RING_ALLOC_CMPL:
5778                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5779                 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
5780                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5781                         /* Association of cp ring with nq */
5782                         grp_info = &bp->grp_info[map_index];
5783                         req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5784                         req->cq_handle = cpu_to_le64(ring->handle);
5785                         req->enables |= cpu_to_le32(
5786                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5787                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5788                         req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5789                 }
5790                 break;
5791         case HWRM_RING_ALLOC_NQ:
5792                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5793                 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
5794                 if (bp->flags & BNXT_FLAG_USING_MSIX)
5795                         req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5796                 break;
5797         default:
5798                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5799                            ring_type);
5800                 return -1;
5801         }
5802
5803         resp = hwrm_req_hold(bp, req);
5804         rc = hwrm_req_send(bp, req);
5805         err = le16_to_cpu(resp->error_code);
5806         ring_id = le16_to_cpu(resp->ring_id);
5807         hwrm_req_drop(bp, req);
5808
5809 exit:
5810         if (rc || err) {
5811                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5812                            ring_type, rc, err);
5813                 return -EIO;
5814         }
5815         ring->fw_ring_id = ring_id;
5816         return rc;
5817 }
5818
5819 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5820 {
5821         int rc;
5822
5823         if (BNXT_PF(bp)) {
5824                 struct hwrm_func_cfg_input *req;
5825
5826                 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
5827                 if (rc)
5828                         return rc;
5829
5830                 req->fid = cpu_to_le16(0xffff);
5831                 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5832                 req->async_event_cr = cpu_to_le16(idx);
5833                 return hwrm_req_send(bp, req);
5834         } else {
5835                 struct hwrm_func_vf_cfg_input *req;
5836
5837                 rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG);
5838                 if (rc)
5839                         return rc;
5840
5841                 req->enables =
5842                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5843                 req->async_event_cr = cpu_to_le16(idx);
5844                 return hwrm_req_send(bp, req);
5845         }
5846 }
5847
5848 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5849                         u32 map_idx, u32 xid)
5850 {
5851         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5852                 if (BNXT_PF(bp))
5853                         db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5854                 else
5855                         db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5856                 switch (ring_type) {
5857                 case HWRM_RING_ALLOC_TX:
5858                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5859                         break;
5860                 case HWRM_RING_ALLOC_RX:
5861                 case HWRM_RING_ALLOC_AGG:
5862                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5863                         break;
5864                 case HWRM_RING_ALLOC_CMPL:
5865                         db->db_key64 = DBR_PATH_L2;
5866                         break;
5867                 case HWRM_RING_ALLOC_NQ:
5868                         db->db_key64 = DBR_PATH_L2;
5869                         break;
5870                 }
5871                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5872         } else {
5873                 db->doorbell = bp->bar1 + map_idx * 0x80;
5874                 switch (ring_type) {
5875                 case HWRM_RING_ALLOC_TX:
5876                         db->db_key32 = DB_KEY_TX;
5877                         break;
5878                 case HWRM_RING_ALLOC_RX:
5879                 case HWRM_RING_ALLOC_AGG:
5880                         db->db_key32 = DB_KEY_RX;
5881                         break;
5882                 case HWRM_RING_ALLOC_CMPL:
5883                         db->db_key32 = DB_KEY_CP;
5884                         break;
5885                 }
5886         }
5887 }
5888
5889 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5890 {
5891         bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5892         int i, rc = 0;
5893         u32 type;
5894
5895         if (bp->flags & BNXT_FLAG_CHIP_P5)
5896                 type = HWRM_RING_ALLOC_NQ;
5897         else
5898                 type = HWRM_RING_ALLOC_CMPL;
5899         for (i = 0; i < bp->cp_nr_rings; i++) {
5900                 struct bnxt_napi *bnapi = bp->bnapi[i];
5901                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5902                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5903                 u32 map_idx = ring->map_idx;
5904                 unsigned int vector;
5905
5906                 vector = bp->irq_tbl[map_idx].vector;
5907                 disable_irq_nosync(vector);
5908                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5909                 if (rc) {
5910                         enable_irq(vector);
5911                         goto err_out;
5912                 }
5913                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5914                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5915                 enable_irq(vector);
5916                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5917
5918                 if (!i) {
5919                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5920                         if (rc)
5921                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5922                 }
5923         }
5924
5925         type = HWRM_RING_ALLOC_TX;
5926         for (i = 0; i < bp->tx_nr_rings; i++) {
5927                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5928                 struct bnxt_ring_struct *ring;
5929                 u32 map_idx;
5930
5931                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5932                         struct bnxt_napi *bnapi = txr->bnapi;
5933                         struct bnxt_cp_ring_info *cpr, *cpr2;
5934                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5935
5936                         cpr = &bnapi->cp_ring;
5937                         cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5938                         ring = &cpr2->cp_ring_struct;
5939                         ring->handle = BNXT_TX_HDL;
5940                         map_idx = bnapi->index;
5941                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5942                         if (rc)
5943                                 goto err_out;
5944                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5945                                     ring->fw_ring_id);
5946                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5947                 }
5948                 ring = &txr->tx_ring_struct;
5949                 map_idx = i;
5950                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5951                 if (rc)
5952                         goto err_out;
5953                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5954         }
5955
5956         type = HWRM_RING_ALLOC_RX;
5957         for (i = 0; i < bp->rx_nr_rings; i++) {
5958                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5959                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5960                 struct bnxt_napi *bnapi = rxr->bnapi;
5961                 u32 map_idx = bnapi->index;
5962
5963                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5964                 if (rc)
5965                         goto err_out;
5966                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5967                 /* If we have agg rings, post agg buffers first. */
5968                 if (!agg_rings)
5969                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5970                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5971                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5972                         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5973                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5974                         struct bnxt_cp_ring_info *cpr2;
5975
5976                         cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5977                         ring = &cpr2->cp_ring_struct;
5978                         ring->handle = BNXT_RX_HDL;
5979                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5980                         if (rc)
5981                                 goto err_out;
5982                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5983                                     ring->fw_ring_id);
5984                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5985                 }
5986         }
5987
5988         if (agg_rings) {
5989                 type = HWRM_RING_ALLOC_AGG;
5990                 for (i = 0; i < bp->rx_nr_rings; i++) {
5991                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5992                         struct bnxt_ring_struct *ring =
5993                                                 &rxr->rx_agg_ring_struct;
5994                         u32 grp_idx = ring->grp_idx;
5995                         u32 map_idx = grp_idx + bp->rx_nr_rings;
5996
5997                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5998                         if (rc)
5999                                 goto err_out;
6000
6001                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
6002                                     ring->fw_ring_id);
6003                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
6004                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
6005                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
6006                 }
6007         }
6008 err_out:
6009         return rc;
6010 }
6011
6012 static int hwrm_ring_free_send_msg(struct bnxt *bp,
6013                                    struct bnxt_ring_struct *ring,
6014                                    u32 ring_type, int cmpl_ring_id)
6015 {
6016         struct hwrm_ring_free_output *resp;
6017         struct hwrm_ring_free_input *req;
6018         u16 error_code = 0;
6019         int rc;
6020
6021         if (BNXT_NO_FW_ACCESS(bp))
6022                 return 0;
6023
6024         rc = hwrm_req_init(bp, req, HWRM_RING_FREE);
6025         if (rc)
6026                 goto exit;
6027
6028         req->cmpl_ring = cpu_to_le16(cmpl_ring_id);
6029         req->ring_type = ring_type;
6030         req->ring_id = cpu_to_le16(ring->fw_ring_id);
6031
6032         resp = hwrm_req_hold(bp, req);
6033         rc = hwrm_req_send(bp, req);
6034         error_code = le16_to_cpu(resp->error_code);
6035         hwrm_req_drop(bp, req);
6036 exit:
6037         if (rc || error_code) {
6038                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
6039                            ring_type, rc, error_code);
6040                 return -EIO;
6041         }
6042         return 0;
6043 }
6044
6045 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
6046 {
6047         u32 type;
6048         int i;
6049
6050         if (!bp->bnapi)
6051                 return;
6052
6053         for (i = 0; i < bp->tx_nr_rings; i++) {
6054                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
6055                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
6056
6057                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6058                         u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
6059
6060                         hwrm_ring_free_send_msg(bp, ring,
6061                                                 RING_FREE_REQ_RING_TYPE_TX,
6062                                                 close_path ? cmpl_ring_id :
6063                                                 INVALID_HW_RING_ID);
6064                         ring->fw_ring_id = INVALID_HW_RING_ID;
6065                 }
6066         }
6067
6068         for (i = 0; i < bp->rx_nr_rings; i++) {
6069                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6070                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
6071                 u32 grp_idx = rxr->bnapi->index;
6072
6073                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6074                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6075
6076                         hwrm_ring_free_send_msg(bp, ring,
6077                                                 RING_FREE_REQ_RING_TYPE_RX,
6078                                                 close_path ? cmpl_ring_id :
6079                                                 INVALID_HW_RING_ID);
6080                         ring->fw_ring_id = INVALID_HW_RING_ID;
6081                         bp->grp_info[grp_idx].rx_fw_ring_id =
6082                                 INVALID_HW_RING_ID;
6083                 }
6084         }
6085
6086         if (bp->flags & BNXT_FLAG_CHIP_P5)
6087                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
6088         else
6089                 type = RING_FREE_REQ_RING_TYPE_RX;
6090         for (i = 0; i < bp->rx_nr_rings; i++) {
6091                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6092                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
6093                 u32 grp_idx = rxr->bnapi->index;
6094
6095                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6096                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6097
6098                         hwrm_ring_free_send_msg(bp, ring, type,
6099                                                 close_path ? cmpl_ring_id :
6100                                                 INVALID_HW_RING_ID);
6101                         ring->fw_ring_id = INVALID_HW_RING_ID;
6102                         bp->grp_info[grp_idx].agg_fw_ring_id =
6103                                 INVALID_HW_RING_ID;
6104                 }
6105         }
6106
6107         /* The completion rings are about to be freed.  After that the
6108          * IRQ doorbell will not work anymore.  So we need to disable
6109          * IRQ here.
6110          */
6111         bnxt_disable_int_sync(bp);
6112
6113         if (bp->flags & BNXT_FLAG_CHIP_P5)
6114                 type = RING_FREE_REQ_RING_TYPE_NQ;
6115         else
6116                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
6117         for (i = 0; i < bp->cp_nr_rings; i++) {
6118                 struct bnxt_napi *bnapi = bp->bnapi[i];
6119                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6120                 struct bnxt_ring_struct *ring;
6121                 int j;
6122
6123                 for (j = 0; j < 2; j++) {
6124                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
6125
6126                         if (cpr2) {
6127                                 ring = &cpr2->cp_ring_struct;
6128                                 if (ring->fw_ring_id == INVALID_HW_RING_ID)
6129                                         continue;
6130                                 hwrm_ring_free_send_msg(bp, ring,
6131                                         RING_FREE_REQ_RING_TYPE_L2_CMPL,
6132                                         INVALID_HW_RING_ID);
6133                                 ring->fw_ring_id = INVALID_HW_RING_ID;
6134                         }
6135                 }
6136                 ring = &cpr->cp_ring_struct;
6137                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6138                         hwrm_ring_free_send_msg(bp, ring, type,
6139                                                 INVALID_HW_RING_ID);
6140                         ring->fw_ring_id = INVALID_HW_RING_ID;
6141                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
6142                 }
6143         }
6144 }
6145
6146 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
6147                            bool shared);
6148
6149 static int bnxt_hwrm_get_rings(struct bnxt *bp)
6150 {
6151         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6152         struct hwrm_func_qcfg_output *resp;
6153         struct hwrm_func_qcfg_input *req;
6154         int rc;
6155
6156         if (bp->hwrm_spec_code < 0x10601)
6157                 return 0;
6158
6159         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6160         if (rc)
6161                 return rc;
6162
6163         req->fid = cpu_to_le16(0xffff);
6164         resp = hwrm_req_hold(bp, req);
6165         rc = hwrm_req_send(bp, req);
6166         if (rc) {
6167                 hwrm_req_drop(bp, req);
6168                 return rc;
6169         }
6170
6171         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6172         if (BNXT_NEW_RM(bp)) {
6173                 u16 cp, stats;
6174
6175                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
6176                 hw_resc->resv_hw_ring_grps =
6177                         le32_to_cpu(resp->alloc_hw_ring_grps);
6178                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
6179                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
6180                 stats = le16_to_cpu(resp->alloc_stat_ctx);
6181                 hw_resc->resv_irqs = cp;
6182                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6183                         int rx = hw_resc->resv_rx_rings;
6184                         int tx = hw_resc->resv_tx_rings;
6185
6186                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6187                                 rx >>= 1;
6188                         if (cp < (rx + tx)) {
6189                                 bnxt_trim_rings(bp, &rx, &tx, cp, false);
6190                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6191                                         rx <<= 1;
6192                                 hw_resc->resv_rx_rings = rx;
6193                                 hw_resc->resv_tx_rings = tx;
6194                         }
6195                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
6196                         hw_resc->resv_hw_ring_grps = rx;
6197                 }
6198                 hw_resc->resv_cp_rings = cp;
6199                 hw_resc->resv_stat_ctxs = stats;
6200         }
6201         hwrm_req_drop(bp, req);
6202         return 0;
6203 }
6204
6205 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
6206 {
6207         struct hwrm_func_qcfg_output *resp;
6208         struct hwrm_func_qcfg_input *req;
6209         int rc;
6210
6211         if (bp->hwrm_spec_code < 0x10601)
6212                 return 0;
6213
6214         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6215         if (rc)
6216                 return rc;
6217
6218         req->fid = cpu_to_le16(fid);
6219         resp = hwrm_req_hold(bp, req);
6220         rc = hwrm_req_send(bp, req);
6221         if (!rc)
6222                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6223
6224         hwrm_req_drop(bp, req);
6225         return rc;
6226 }
6227
6228 static bool bnxt_rfs_supported(struct bnxt *bp);
6229
6230 static struct hwrm_func_cfg_input *
6231 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6232                              int ring_grps, int cp_rings, int stats, int vnics)
6233 {
6234         struct hwrm_func_cfg_input *req;
6235         u32 enables = 0;
6236
6237         if (hwrm_req_init(bp, req, HWRM_FUNC_CFG))
6238                 return NULL;
6239
6240         req->fid = cpu_to_le16(0xffff);
6241         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6242         req->num_tx_rings = cpu_to_le16(tx_rings);
6243         if (BNXT_NEW_RM(bp)) {
6244                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
6245                 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6246                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6247                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
6248                         enables |= tx_rings + ring_grps ?
6249                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6250                         enables |= rx_rings ?
6251                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6252                 } else {
6253                         enables |= cp_rings ?
6254                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6255                         enables |= ring_grps ?
6256                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
6257                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6258                 }
6259                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
6260
6261                 req->num_rx_rings = cpu_to_le16(rx_rings);
6262                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6263                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6264                         req->num_msix = cpu_to_le16(cp_rings);
6265                         req->num_rsscos_ctxs =
6266                                 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6267                 } else {
6268                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
6269                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6270                         req->num_rsscos_ctxs = cpu_to_le16(1);
6271                         if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
6272                             bnxt_rfs_supported(bp))
6273                                 req->num_rsscos_ctxs =
6274                                         cpu_to_le16(ring_grps + 1);
6275                 }
6276                 req->num_stat_ctxs = cpu_to_le16(stats);
6277                 req->num_vnics = cpu_to_le16(vnics);
6278         }
6279         req->enables = cpu_to_le32(enables);
6280         return req;
6281 }
6282
6283 static struct hwrm_func_vf_cfg_input *
6284 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6285                              int ring_grps, int cp_rings, int stats, int vnics)
6286 {
6287         struct hwrm_func_vf_cfg_input *req;
6288         u32 enables = 0;
6289
6290         if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG))
6291                 return NULL;
6292
6293         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6294         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
6295                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6296         enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6297         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6298                 enables |= tx_rings + ring_grps ?
6299                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6300         } else {
6301                 enables |= cp_rings ?
6302                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6303                 enables |= ring_grps ?
6304                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
6305         }
6306         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
6307         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
6308
6309         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6310         req->num_tx_rings = cpu_to_le16(tx_rings);
6311         req->num_rx_rings = cpu_to_le16(rx_rings);
6312         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6313                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6314                 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6315         } else {
6316                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
6317                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6318                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6319         }
6320         req->num_stat_ctxs = cpu_to_le16(stats);
6321         req->num_vnics = cpu_to_le16(vnics);
6322
6323         req->enables = cpu_to_le32(enables);
6324         return req;
6325 }
6326
6327 static int
6328 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6329                            int ring_grps, int cp_rings, int stats, int vnics)
6330 {
6331         struct hwrm_func_cfg_input *req;
6332         int rc;
6333
6334         req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
6335                                            cp_rings, stats, vnics);
6336         if (!req)
6337                 return -ENOMEM;
6338
6339         if (!req->enables) {
6340                 hwrm_req_drop(bp, req);
6341                 return 0;
6342         }
6343
6344         rc = hwrm_req_send(bp, req);
6345         if (rc)
6346                 return rc;
6347
6348         if (bp->hwrm_spec_code < 0x10601)
6349                 bp->hw_resc.resv_tx_rings = tx_rings;
6350
6351         return bnxt_hwrm_get_rings(bp);
6352 }
6353
6354 static int
6355 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6356                            int ring_grps, int cp_rings, int stats, int vnics)
6357 {
6358         struct hwrm_func_vf_cfg_input *req;
6359         int rc;
6360
6361         if (!BNXT_NEW_RM(bp)) {
6362                 bp->hw_resc.resv_tx_rings = tx_rings;
6363                 return 0;
6364         }
6365
6366         req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6367                                            cp_rings, stats, vnics);
6368         if (!req)
6369                 return -ENOMEM;
6370
6371         rc = hwrm_req_send(bp, req);
6372         if (rc)
6373                 return rc;
6374
6375         return bnxt_hwrm_get_rings(bp);
6376 }
6377
6378 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6379                                    int cp, int stat, int vnic)
6380 {
6381         if (BNXT_PF(bp))
6382                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6383                                                   vnic);
6384         else
6385                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6386                                                   vnic);
6387 }
6388
6389 int bnxt_nq_rings_in_use(struct bnxt *bp)
6390 {
6391         int cp = bp->cp_nr_rings;
6392         int ulp_msix, ulp_base;
6393
6394         ulp_msix = bnxt_get_ulp_msix_num(bp);
6395         if (ulp_msix) {
6396                 ulp_base = bnxt_get_ulp_msix_base(bp);
6397                 cp += ulp_msix;
6398                 if ((ulp_base + ulp_msix) > cp)
6399                         cp = ulp_base + ulp_msix;
6400         }
6401         return cp;
6402 }
6403
6404 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6405 {
6406         int cp;
6407
6408         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6409                 return bnxt_nq_rings_in_use(bp);
6410
6411         cp = bp->tx_nr_rings + bp->rx_nr_rings;
6412         return cp;
6413 }
6414
6415 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6416 {
6417         int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6418         int cp = bp->cp_nr_rings;
6419
6420         if (!ulp_stat)
6421                 return cp;
6422
6423         if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6424                 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6425
6426         return cp + ulp_stat;
6427 }
6428
6429 /* Check if a default RSS map needs to be setup.  This function is only
6430  * used on older firmware that does not require reserving RX rings.
6431  */
6432 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6433 {
6434         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6435
6436         /* The RSS map is valid for RX rings set to resv_rx_rings */
6437         if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6438                 hw_resc->resv_rx_rings = bp->rx_nr_rings;
6439                 if (!netif_is_rxfh_configured(bp->dev))
6440                         bnxt_set_dflt_rss_indir_tbl(bp);
6441         }
6442 }
6443
6444 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6445 {
6446         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6447         int cp = bnxt_cp_rings_in_use(bp);
6448         int nq = bnxt_nq_rings_in_use(bp);
6449         int rx = bp->rx_nr_rings, stat;
6450         int vnic = 1, grp = rx;
6451
6452         if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6453             bp->hwrm_spec_code >= 0x10601)
6454                 return true;
6455
6456         /* Old firmware does not need RX ring reservations but we still
6457          * need to setup a default RSS map when needed.  With new firmware
6458          * we go through RX ring reservations first and then set up the
6459          * RSS map for the successfully reserved RX rings when needed.
6460          */
6461         if (!BNXT_NEW_RM(bp)) {
6462                 bnxt_check_rss_tbl_no_rmgr(bp);
6463                 return false;
6464         }
6465         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6466                 vnic = rx + 1;
6467         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6468                 rx <<= 1;
6469         stat = bnxt_get_func_stat_ctxs(bp);
6470         if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6471             hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6472             (hw_resc->resv_hw_ring_grps != grp &&
6473              !(bp->flags & BNXT_FLAG_CHIP_P5)))
6474                 return true;
6475         if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6476             hw_resc->resv_irqs != nq)
6477                 return true;
6478         return false;
6479 }
6480
6481 static int __bnxt_reserve_rings(struct bnxt *bp)
6482 {
6483         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6484         int cp = bnxt_nq_rings_in_use(bp);
6485         int tx = bp->tx_nr_rings;
6486         int rx = bp->rx_nr_rings;
6487         int grp, rx_rings, rc;
6488         int vnic = 1, stat;
6489         bool sh = false;
6490
6491         if (!bnxt_need_reserve_rings(bp))
6492                 return 0;
6493
6494         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6495                 sh = true;
6496         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6497                 vnic = rx + 1;
6498         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6499                 rx <<= 1;
6500         grp = bp->rx_nr_rings;
6501         stat = bnxt_get_func_stat_ctxs(bp);
6502
6503         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6504         if (rc)
6505                 return rc;
6506
6507         tx = hw_resc->resv_tx_rings;
6508         if (BNXT_NEW_RM(bp)) {
6509                 rx = hw_resc->resv_rx_rings;
6510                 cp = hw_resc->resv_irqs;
6511                 grp = hw_resc->resv_hw_ring_grps;
6512                 vnic = hw_resc->resv_vnics;
6513                 stat = hw_resc->resv_stat_ctxs;
6514         }
6515
6516         rx_rings = rx;
6517         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6518                 if (rx >= 2) {
6519                         rx_rings = rx >> 1;
6520                 } else {
6521                         if (netif_running(bp->dev))
6522                                 return -ENOMEM;
6523
6524                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6525                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6526                         bp->dev->hw_features &= ~NETIF_F_LRO;
6527                         bp->dev->features &= ~NETIF_F_LRO;
6528                         bnxt_set_ring_params(bp);
6529                 }
6530         }
6531         rx_rings = min_t(int, rx_rings, grp);
6532         cp = min_t(int, cp, bp->cp_nr_rings);
6533         if (stat > bnxt_get_ulp_stat_ctxs(bp))
6534                 stat -= bnxt_get_ulp_stat_ctxs(bp);
6535         cp = min_t(int, cp, stat);
6536         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6537         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6538                 rx = rx_rings << 1;
6539         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6540         bp->tx_nr_rings = tx;
6541
6542         /* If we cannot reserve all the RX rings, reset the RSS map only
6543          * if absolutely necessary
6544          */
6545         if (rx_rings != bp->rx_nr_rings) {
6546                 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6547                             rx_rings, bp->rx_nr_rings);
6548                 if (netif_is_rxfh_configured(bp->dev) &&
6549                     (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6550                      bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6551                      bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6552                         netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6553                         bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6554                 }
6555         }
6556         bp->rx_nr_rings = rx_rings;
6557         bp->cp_nr_rings = cp;
6558
6559         if (!tx || !rx || !cp || !grp || !vnic || !stat)
6560                 return -ENOMEM;
6561
6562         if (!netif_is_rxfh_configured(bp->dev))
6563                 bnxt_set_dflt_rss_indir_tbl(bp);
6564
6565         return rc;
6566 }
6567
6568 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6569                                     int ring_grps, int cp_rings, int stats,
6570                                     int vnics)
6571 {
6572         struct hwrm_func_vf_cfg_input *req;
6573         u32 flags;
6574
6575         if (!BNXT_NEW_RM(bp))
6576                 return 0;
6577
6578         req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6579                                            cp_rings, stats, vnics);
6580         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6581                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6582                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6583                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6584                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6585                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6586         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6587                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6588
6589         req->flags = cpu_to_le32(flags);
6590         return hwrm_req_send_silent(bp, req);
6591 }
6592
6593 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6594                                     int ring_grps, int cp_rings, int stats,
6595                                     int vnics)
6596 {
6597         struct hwrm_func_cfg_input *req;
6598         u32 flags;
6599
6600         req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
6601                                            cp_rings, stats, vnics);
6602         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6603         if (BNXT_NEW_RM(bp)) {
6604                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6605                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6606                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6607                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6608                 if (bp->flags & BNXT_FLAG_CHIP_P5)
6609                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6610                                  FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6611                 else
6612                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6613         }
6614
6615         req->flags = cpu_to_le32(flags);
6616         return hwrm_req_send_silent(bp, req);
6617 }
6618
6619 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6620                                  int ring_grps, int cp_rings, int stats,
6621                                  int vnics)
6622 {
6623         if (bp->hwrm_spec_code < 0x10801)
6624                 return 0;
6625
6626         if (BNXT_PF(bp))
6627                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6628                                                 ring_grps, cp_rings, stats,
6629                                                 vnics);
6630
6631         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6632                                         cp_rings, stats, vnics);
6633 }
6634
6635 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6636 {
6637         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6638         struct hwrm_ring_aggint_qcaps_output *resp;
6639         struct hwrm_ring_aggint_qcaps_input *req;
6640         int rc;
6641
6642         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6643         coal_cap->num_cmpl_dma_aggr_max = 63;
6644         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6645         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6646         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6647         coal_cap->int_lat_tmr_min_max = 65535;
6648         coal_cap->int_lat_tmr_max_max = 65535;
6649         coal_cap->num_cmpl_aggr_int_max = 65535;
6650         coal_cap->timer_units = 80;
6651
6652         if (bp->hwrm_spec_code < 0x10902)
6653                 return;
6654
6655         if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS))
6656                 return;
6657
6658         resp = hwrm_req_hold(bp, req);
6659         rc = hwrm_req_send_silent(bp, req);
6660         if (!rc) {
6661                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6662                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6663                 coal_cap->num_cmpl_dma_aggr_max =
6664                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6665                 coal_cap->num_cmpl_dma_aggr_during_int_max =
6666                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6667                 coal_cap->cmpl_aggr_dma_tmr_max =
6668                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6669                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6670                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6671                 coal_cap->int_lat_tmr_min_max =
6672                         le16_to_cpu(resp->int_lat_tmr_min_max);
6673                 coal_cap->int_lat_tmr_max_max =
6674                         le16_to_cpu(resp->int_lat_tmr_max_max);
6675                 coal_cap->num_cmpl_aggr_int_max =
6676                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
6677                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6678         }
6679         hwrm_req_drop(bp, req);
6680 }
6681
6682 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6683 {
6684         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6685
6686         return usec * 1000 / coal_cap->timer_units;
6687 }
6688
6689 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6690         struct bnxt_coal *hw_coal,
6691         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6692 {
6693         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6694         u16 val, tmr, max, flags = hw_coal->flags;
6695         u32 cmpl_params = coal_cap->cmpl_params;
6696
6697         max = hw_coal->bufs_per_record * 128;
6698         if (hw_coal->budget)
6699                 max = hw_coal->bufs_per_record * hw_coal->budget;
6700         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6701
6702         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6703         req->num_cmpl_aggr_int = cpu_to_le16(val);
6704
6705         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6706         req->num_cmpl_dma_aggr = cpu_to_le16(val);
6707
6708         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6709                       coal_cap->num_cmpl_dma_aggr_during_int_max);
6710         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6711
6712         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6713         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6714         req->int_lat_tmr_max = cpu_to_le16(tmr);
6715
6716         /* min timer set to 1/2 of interrupt timer */
6717         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6718                 val = tmr / 2;
6719                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6720                 req->int_lat_tmr_min = cpu_to_le16(val);
6721                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6722         }
6723
6724         /* buf timer set to 1/4 of interrupt timer */
6725         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6726         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6727
6728         if (cmpl_params &
6729             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6730                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6731                 val = clamp_t(u16, tmr, 1,
6732                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6733                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6734                 req->enables |=
6735                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6736         }
6737
6738         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6739             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6740                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6741         req->flags = cpu_to_le16(flags);
6742         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6743 }
6744
6745 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6746                                    struct bnxt_coal *hw_coal)
6747 {
6748         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req;
6749         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6750         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6751         u32 nq_params = coal_cap->nq_params;
6752         u16 tmr;
6753         int rc;
6754
6755         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6756                 return 0;
6757
6758         rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6759         if (rc)
6760                 return rc;
6761
6762         req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6763         req->flags =
6764                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6765
6766         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6767         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6768         req->int_lat_tmr_min = cpu_to_le16(tmr);
6769         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6770         return hwrm_req_send(bp, req);
6771 }
6772
6773 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6774 {
6775         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx;
6776         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6777         struct bnxt_coal coal;
6778         int rc;
6779
6780         /* Tick values in micro seconds.
6781          * 1 coal_buf x bufs_per_record = 1 completion record.
6782          */
6783         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6784
6785         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6786         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6787
6788         if (!bnapi->rx_ring)
6789                 return -ENODEV;
6790
6791         rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6792         if (rc)
6793                 return rc;
6794
6795         bnxt_hwrm_set_coal_params(bp, &coal, req_rx);
6796
6797         req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6798
6799         return hwrm_req_send(bp, req_rx);
6800 }
6801
6802 int bnxt_hwrm_set_coal(struct bnxt *bp)
6803 {
6804         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx,
6805                                                            *req;
6806         int i, rc;
6807
6808         rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6809         if (rc)
6810                 return rc;
6811
6812         rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6813         if (rc) {
6814                 hwrm_req_drop(bp, req_rx);
6815                 return rc;
6816         }
6817
6818         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx);
6819         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx);
6820
6821         hwrm_req_hold(bp, req_rx);
6822         hwrm_req_hold(bp, req_tx);
6823         for (i = 0; i < bp->cp_nr_rings; i++) {
6824                 struct bnxt_napi *bnapi = bp->bnapi[i];
6825                 struct bnxt_coal *hw_coal;
6826                 u16 ring_id;
6827
6828                 req = req_rx;
6829                 if (!bnapi->rx_ring) {
6830                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6831                         req = req_tx;
6832                 } else {
6833                         ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6834                 }
6835                 req->ring_id = cpu_to_le16(ring_id);
6836
6837                 rc = hwrm_req_send(bp, req);
6838                 if (rc)
6839                         break;
6840
6841                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6842                         continue;
6843
6844                 if (bnapi->rx_ring && bnapi->tx_ring) {
6845                         req = req_tx;
6846                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6847                         req->ring_id = cpu_to_le16(ring_id);
6848                         rc = hwrm_req_send(bp, req);
6849                         if (rc)
6850                                 break;
6851                 }
6852                 if (bnapi->rx_ring)
6853                         hw_coal = &bp->rx_coal;
6854                 else
6855                         hw_coal = &bp->tx_coal;
6856                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6857         }
6858         hwrm_req_drop(bp, req_rx);
6859         hwrm_req_drop(bp, req_tx);
6860         return rc;
6861 }
6862
6863 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6864 {
6865         struct hwrm_stat_ctx_clr_stats_input *req0 = NULL;
6866         struct hwrm_stat_ctx_free_input *req;
6867         int i;
6868
6869         if (!bp->bnapi)
6870                 return;
6871
6872         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6873                 return;
6874
6875         if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE))
6876                 return;
6877         if (BNXT_FW_MAJ(bp) <= 20) {
6878                 if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) {
6879                         hwrm_req_drop(bp, req);
6880                         return;
6881                 }
6882                 hwrm_req_hold(bp, req0);
6883         }
6884         hwrm_req_hold(bp, req);
6885         for (i = 0; i < bp->cp_nr_rings; i++) {
6886                 struct bnxt_napi *bnapi = bp->bnapi[i];
6887                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6888
6889                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6890                         req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6891                         if (req0) {
6892                                 req0->stat_ctx_id = req->stat_ctx_id;
6893                                 hwrm_req_send(bp, req0);
6894                         }
6895                         hwrm_req_send(bp, req);
6896
6897                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6898                 }
6899         }
6900         hwrm_req_drop(bp, req);
6901         if (req0)
6902                 hwrm_req_drop(bp, req0);
6903 }
6904
6905 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6906 {
6907         struct hwrm_stat_ctx_alloc_output *resp;
6908         struct hwrm_stat_ctx_alloc_input *req;
6909         int rc, i;
6910
6911         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6912                 return 0;
6913
6914         rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC);
6915         if (rc)
6916                 return rc;
6917
6918         req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6919         req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6920
6921         resp = hwrm_req_hold(bp, req);
6922         for (i = 0; i < bp->cp_nr_rings; i++) {
6923                 struct bnxt_napi *bnapi = bp->bnapi[i];
6924                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6925
6926                 req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6927
6928                 rc = hwrm_req_send(bp, req);
6929                 if (rc)
6930                         break;
6931
6932                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6933
6934                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6935         }
6936         hwrm_req_drop(bp, req);
6937         return rc;
6938 }
6939
6940 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6941 {
6942         struct hwrm_func_qcfg_output *resp;
6943         struct hwrm_func_qcfg_input *req;
6944         u32 min_db_offset = 0;
6945         u16 flags;
6946         int rc;
6947
6948         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6949         if (rc)
6950                 return rc;
6951
6952         req->fid = cpu_to_le16(0xffff);
6953         resp = hwrm_req_hold(bp, req);
6954         rc = hwrm_req_send(bp, req);
6955         if (rc)
6956                 goto func_qcfg_exit;
6957
6958 #ifdef CONFIG_BNXT_SRIOV
6959         if (BNXT_VF(bp)) {
6960                 struct bnxt_vf_info *vf = &bp->vf;
6961
6962                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6963         } else {
6964                 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6965         }
6966 #endif
6967         flags = le16_to_cpu(resp->flags);
6968         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6969                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6970                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6971                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6972                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6973         }
6974         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6975                 bp->flags |= BNXT_FLAG_MULTI_HOST;
6976
6977         if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
6978                 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
6979
6980         switch (resp->port_partition_type) {
6981         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6982         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6983         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6984                 bp->port_partition_type = resp->port_partition_type;
6985                 break;
6986         }
6987         if (bp->hwrm_spec_code < 0x10707 ||
6988             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6989                 bp->br_mode = BRIDGE_MODE_VEB;
6990         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6991                 bp->br_mode = BRIDGE_MODE_VEPA;
6992         else
6993                 bp->br_mode = BRIDGE_MODE_UNDEF;
6994
6995         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6996         if (!bp->max_mtu)
6997                 bp->max_mtu = BNXT_MAX_MTU;
6998
6999         if (bp->db_size)
7000                 goto func_qcfg_exit;
7001
7002         if (bp->flags & BNXT_FLAG_CHIP_P5) {
7003                 if (BNXT_PF(bp))
7004                         min_db_offset = DB_PF_OFFSET_P5;
7005                 else
7006                         min_db_offset = DB_VF_OFFSET_P5;
7007         }
7008         bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
7009                                  1024);
7010         if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
7011             bp->db_size <= min_db_offset)
7012                 bp->db_size = pci_resource_len(bp->pdev, 2);
7013
7014 func_qcfg_exit:
7015         hwrm_req_drop(bp, req);
7016         return rc;
7017 }
7018
7019 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx,
7020                         struct hwrm_func_backing_store_qcaps_output *resp)
7021 {
7022         struct bnxt_mem_init *mem_init;
7023         u16 init_mask;
7024         u8 init_val;
7025         u8 *offset;
7026         int i;
7027
7028         init_val = resp->ctx_kind_initializer;
7029         init_mask = le16_to_cpu(resp->ctx_init_mask);
7030         offset = &resp->qp_init_offset;
7031         mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
7032         for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) {
7033                 mem_init->init_val = init_val;
7034                 mem_init->offset = BNXT_MEM_INVALID_OFFSET;
7035                 if (!init_mask)
7036                         continue;
7037                 if (i == BNXT_CTX_MEM_INIT_STAT)
7038                         offset = &resp->stat_init_offset;
7039                 if (init_mask & (1 << i))
7040                         mem_init->offset = *offset * 4;
7041                 else
7042                         mem_init->init_val = 0;
7043         }
7044         ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size;
7045         ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size;
7046         ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size;
7047         ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size;
7048         ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size;
7049         ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size;
7050 }
7051
7052 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
7053 {
7054         struct hwrm_func_backing_store_qcaps_output *resp;
7055         struct hwrm_func_backing_store_qcaps_input *req;
7056         int rc;
7057
7058         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
7059                 return 0;
7060
7061         rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS);
7062         if (rc)
7063                 return rc;
7064
7065         resp = hwrm_req_hold(bp, req);
7066         rc = hwrm_req_send_silent(bp, req);
7067         if (!rc) {
7068                 struct bnxt_ctx_pg_info *ctx_pg;
7069                 struct bnxt_ctx_mem_info *ctx;
7070                 int i, tqm_rings;
7071
7072                 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
7073                 if (!ctx) {
7074                         rc = -ENOMEM;
7075                         goto ctx_err;
7076                 }
7077                 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
7078                 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
7079                 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
7080                 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
7081                 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
7082                 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
7083                 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
7084                 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
7085                 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
7086                 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
7087                 ctx->vnic_max_vnic_entries =
7088                         le16_to_cpu(resp->vnic_max_vnic_entries);
7089                 ctx->vnic_max_ring_table_entries =
7090                         le16_to_cpu(resp->vnic_max_ring_table_entries);
7091                 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
7092                 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
7093                 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
7094                 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
7095                 ctx->tqm_min_entries_per_ring =
7096                         le32_to_cpu(resp->tqm_min_entries_per_ring);
7097                 ctx->tqm_max_entries_per_ring =
7098                         le32_to_cpu(resp->tqm_max_entries_per_ring);
7099                 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
7100                 if (!ctx->tqm_entries_multiple)
7101                         ctx->tqm_entries_multiple = 1;
7102                 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
7103                 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
7104                 ctx->mrav_num_entries_units =
7105                         le16_to_cpu(resp->mrav_num_entries_units);
7106                 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
7107                 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
7108
7109                 bnxt_init_ctx_initializer(ctx, resp);
7110
7111                 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
7112                 if (!ctx->tqm_fp_rings_count)
7113                         ctx->tqm_fp_rings_count = bp->max_q;
7114                 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS)
7115                         ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS;
7116
7117                 tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS;
7118                 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
7119                 if (!ctx_pg) {
7120                         kfree(ctx);
7121                         rc = -ENOMEM;
7122                         goto ctx_err;
7123                 }
7124                 for (i = 0; i < tqm_rings; i++, ctx_pg++)
7125                         ctx->tqm_mem[i] = ctx_pg;
7126                 bp->ctx = ctx;
7127         } else {
7128                 rc = 0;
7129         }
7130 ctx_err:
7131         hwrm_req_drop(bp, req);
7132         return rc;
7133 }
7134
7135 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
7136                                   __le64 *pg_dir)
7137 {
7138         if (!rmem->nr_pages)
7139                 return;
7140
7141         BNXT_SET_CTX_PAGE_ATTR(*pg_attr);
7142         if (rmem->depth >= 1) {
7143                 if (rmem->depth == 2)
7144                         *pg_attr |= 2;
7145                 else
7146                         *pg_attr |= 1;
7147                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
7148         } else {
7149                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
7150         }
7151 }
7152
7153 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
7154         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
7155          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
7156          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
7157          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
7158          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
7159
7160 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
7161 {
7162         struct hwrm_func_backing_store_cfg_input *req;
7163         struct bnxt_ctx_mem_info *ctx = bp->ctx;
7164         struct bnxt_ctx_pg_info *ctx_pg;
7165         void **__req = (void **)&req;
7166         u32 req_len = sizeof(*req);
7167         __le32 *num_entries;
7168         __le64 *pg_dir;
7169         u32 flags = 0;
7170         u8 *pg_attr;
7171         u32 ena;
7172         int rc;
7173         int i;
7174
7175         if (!ctx)
7176                 return 0;
7177
7178         if (req_len > bp->hwrm_max_ext_req_len)
7179                 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN;
7180         rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len);
7181         if (rc)
7182                 return rc;
7183
7184         req->enables = cpu_to_le32(enables);
7185         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
7186                 ctx_pg = &ctx->qp_mem;
7187                 req->qp_num_entries = cpu_to_le32(ctx_pg->entries);
7188                 req->qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
7189                 req->qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
7190                 req->qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
7191                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7192                                       &req->qpc_pg_size_qpc_lvl,
7193                                       &req->qpc_page_dir);
7194         }
7195         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
7196                 ctx_pg = &ctx->srq_mem;
7197                 req->srq_num_entries = cpu_to_le32(ctx_pg->entries);
7198                 req->srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
7199                 req->srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
7200                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7201                                       &req->srq_pg_size_srq_lvl,
7202                                       &req->srq_page_dir);
7203         }
7204         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
7205                 ctx_pg = &ctx->cq_mem;
7206                 req->cq_num_entries = cpu_to_le32(ctx_pg->entries);
7207                 req->cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
7208                 req->cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
7209                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7210                                       &req->cq_pg_size_cq_lvl,
7211                                       &req->cq_page_dir);
7212         }
7213         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
7214                 ctx_pg = &ctx->vnic_mem;
7215                 req->vnic_num_vnic_entries =
7216                         cpu_to_le16(ctx->vnic_max_vnic_entries);
7217                 req->vnic_num_ring_table_entries =
7218                         cpu_to_le16(ctx->vnic_max_ring_table_entries);
7219                 req->vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
7220                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7221                                       &req->vnic_pg_size_vnic_lvl,
7222                                       &req->vnic_page_dir);
7223         }
7224         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
7225                 ctx_pg = &ctx->stat_mem;
7226                 req->stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
7227                 req->stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
7228                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7229                                       &req->stat_pg_size_stat_lvl,
7230                                       &req->stat_page_dir);
7231         }
7232         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
7233                 ctx_pg = &ctx->mrav_mem;
7234                 req->mrav_num_entries = cpu_to_le32(ctx_pg->entries);
7235                 if (ctx->mrav_num_entries_units)
7236                         flags |=
7237                         FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
7238                 req->mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
7239                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7240                                       &req->mrav_pg_size_mrav_lvl,
7241                                       &req->mrav_page_dir);
7242         }
7243         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
7244                 ctx_pg = &ctx->tim_mem;
7245                 req->tim_num_entries = cpu_to_le32(ctx_pg->entries);
7246                 req->tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
7247                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7248                                       &req->tim_pg_size_tim_lvl,
7249                                       &req->tim_page_dir);
7250         }
7251         for (i = 0, num_entries = &req->tqm_sp_num_entries,
7252              pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl,
7253              pg_dir = &req->tqm_sp_page_dir,
7254              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
7255              i < BNXT_MAX_TQM_RINGS;
7256              i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
7257                 if (!(enables & ena))
7258                         continue;
7259
7260                 req->tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
7261                 ctx_pg = ctx->tqm_mem[i];
7262                 *num_entries = cpu_to_le32(ctx_pg->entries);
7263                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
7264         }
7265         req->flags = cpu_to_le32(flags);
7266         return hwrm_req_send(bp, req);
7267 }
7268
7269 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
7270                                   struct bnxt_ctx_pg_info *ctx_pg)
7271 {
7272         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7273
7274         rmem->page_size = BNXT_PAGE_SIZE;
7275         rmem->pg_arr = ctx_pg->ctx_pg_arr;
7276         rmem->dma_arr = ctx_pg->ctx_dma_arr;
7277         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
7278         if (rmem->depth >= 1)
7279                 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
7280         return bnxt_alloc_ring(bp, rmem);
7281 }
7282
7283 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
7284                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
7285                                   u8 depth, struct bnxt_mem_init *mem_init)
7286 {
7287         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7288         int rc;
7289
7290         if (!mem_size)
7291                 return -EINVAL;
7292
7293         ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7294         if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
7295                 ctx_pg->nr_pages = 0;
7296                 return -EINVAL;
7297         }
7298         if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
7299                 int nr_tbls, i;
7300
7301                 rmem->depth = 2;
7302                 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
7303                                              GFP_KERNEL);
7304                 if (!ctx_pg->ctx_pg_tbl)
7305                         return -ENOMEM;
7306                 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
7307                 rmem->nr_pages = nr_tbls;
7308                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7309                 if (rc)
7310                         return rc;
7311                 for (i = 0; i < nr_tbls; i++) {
7312                         struct bnxt_ctx_pg_info *pg_tbl;
7313
7314                         pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
7315                         if (!pg_tbl)
7316                                 return -ENOMEM;
7317                         ctx_pg->ctx_pg_tbl[i] = pg_tbl;
7318                         rmem = &pg_tbl->ring_mem;
7319                         rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
7320                         rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
7321                         rmem->depth = 1;
7322                         rmem->nr_pages = MAX_CTX_PAGES;
7323                         rmem->mem_init = mem_init;
7324                         if (i == (nr_tbls - 1)) {
7325                                 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
7326
7327                                 if (rem)
7328                                         rmem->nr_pages = rem;
7329                         }
7330                         rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
7331                         if (rc)
7332                                 break;
7333                 }
7334         } else {
7335                 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7336                 if (rmem->nr_pages > 1 || depth)
7337                         rmem->depth = 1;
7338                 rmem->mem_init = mem_init;
7339                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7340         }
7341         return rc;
7342 }
7343
7344 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
7345                                   struct bnxt_ctx_pg_info *ctx_pg)
7346 {
7347         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7348
7349         if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
7350             ctx_pg->ctx_pg_tbl) {
7351                 int i, nr_tbls = rmem->nr_pages;
7352
7353                 for (i = 0; i < nr_tbls; i++) {
7354                         struct bnxt_ctx_pg_info *pg_tbl;
7355                         struct bnxt_ring_mem_info *rmem2;
7356
7357                         pg_tbl = ctx_pg->ctx_pg_tbl[i];
7358                         if (!pg_tbl)
7359                                 continue;
7360                         rmem2 = &pg_tbl->ring_mem;
7361                         bnxt_free_ring(bp, rmem2);
7362                         ctx_pg->ctx_pg_arr[i] = NULL;
7363                         kfree(pg_tbl);
7364                         ctx_pg->ctx_pg_tbl[i] = NULL;
7365                 }
7366                 kfree(ctx_pg->ctx_pg_tbl);
7367                 ctx_pg->ctx_pg_tbl = NULL;
7368         }
7369         bnxt_free_ring(bp, rmem);
7370         ctx_pg->nr_pages = 0;
7371 }
7372
7373 void bnxt_free_ctx_mem(struct bnxt *bp)
7374 {
7375         struct bnxt_ctx_mem_info *ctx = bp->ctx;
7376         int i;
7377
7378         if (!ctx)
7379                 return;
7380
7381         if (ctx->tqm_mem[0]) {
7382                 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7383                         bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7384                 kfree(ctx->tqm_mem[0]);
7385                 ctx->tqm_mem[0] = NULL;
7386         }
7387
7388         bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7389         bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7390         bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7391         bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7392         bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7393         bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7394         bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7395         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7396 }
7397
7398 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7399 {
7400         struct bnxt_ctx_pg_info *ctx_pg;
7401         struct bnxt_ctx_mem_info *ctx;
7402         struct bnxt_mem_init *init;
7403         u32 mem_size, ena, entries;
7404         u32 entries_sp, min;
7405         u32 num_mr, num_ah;
7406         u32 extra_srqs = 0;
7407         u32 extra_qps = 0;
7408         u8 pg_lvl = 1;
7409         int i, rc;
7410
7411         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7412         if (rc) {
7413                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7414                            rc);
7415                 return rc;
7416         }
7417         ctx = bp->ctx;
7418         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7419                 return 0;
7420
7421         if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7422                 pg_lvl = 2;
7423                 extra_qps = 65536;
7424                 extra_srqs = 8192;
7425         }
7426
7427         ctx_pg = &ctx->qp_mem;
7428         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7429                           extra_qps;
7430         if (ctx->qp_entry_size) {
7431                 mem_size = ctx->qp_entry_size * ctx_pg->entries;
7432                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
7433                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7434                 if (rc)
7435                         return rc;
7436         }
7437
7438         ctx_pg = &ctx->srq_mem;
7439         ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7440         if (ctx->srq_entry_size) {
7441                 mem_size = ctx->srq_entry_size * ctx_pg->entries;
7442                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ];
7443                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7444                 if (rc)
7445                         return rc;
7446         }
7447
7448         ctx_pg = &ctx->cq_mem;
7449         ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7450         if (ctx->cq_entry_size) {
7451                 mem_size = ctx->cq_entry_size * ctx_pg->entries;
7452                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ];
7453                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7454                 if (rc)
7455                         return rc;
7456         }
7457
7458         ctx_pg = &ctx->vnic_mem;
7459         ctx_pg->entries = ctx->vnic_max_vnic_entries +
7460                           ctx->vnic_max_ring_table_entries;
7461         if (ctx->vnic_entry_size) {
7462                 mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7463                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC];
7464                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7465                 if (rc)
7466                         return rc;
7467         }
7468
7469         ctx_pg = &ctx->stat_mem;
7470         ctx_pg->entries = ctx->stat_max_entries;
7471         if (ctx->stat_entry_size) {
7472                 mem_size = ctx->stat_entry_size * ctx_pg->entries;
7473                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT];
7474                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7475                 if (rc)
7476                         return rc;
7477         }
7478
7479         ena = 0;
7480         if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7481                 goto skip_rdma;
7482
7483         ctx_pg = &ctx->mrav_mem;
7484         /* 128K extra is needed to accommodate static AH context
7485          * allocation by f/w.
7486          */
7487         num_mr = 1024 * 256;
7488         num_ah = 1024 * 128;
7489         ctx_pg->entries = num_mr + num_ah;
7490         if (ctx->mrav_entry_size) {
7491                 mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7492                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV];
7493                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init);
7494                 if (rc)
7495                         return rc;
7496         }
7497         ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7498         if (ctx->mrav_num_entries_units)
7499                 ctx_pg->entries =
7500                         ((num_mr / ctx->mrav_num_entries_units) << 16) |
7501                          (num_ah / ctx->mrav_num_entries_units);
7502
7503         ctx_pg = &ctx->tim_mem;
7504         ctx_pg->entries = ctx->qp_mem.entries;
7505         if (ctx->tim_entry_size) {
7506                 mem_size = ctx->tim_entry_size * ctx_pg->entries;
7507                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL);
7508                 if (rc)
7509                         return rc;
7510         }
7511         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7512
7513 skip_rdma:
7514         min = ctx->tqm_min_entries_per_ring;
7515         entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7516                      2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7517         entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7518         entries = ctx->qp_max_l2_entries + 2 * (extra_qps + ctx->qp_min_qp1_entries);
7519         entries = roundup(entries, ctx->tqm_entries_multiple);
7520         entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7521         for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7522                 ctx_pg = ctx->tqm_mem[i];
7523                 ctx_pg->entries = i ? entries : entries_sp;
7524                 if (ctx->tqm_entry_size) {
7525                         mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7526                         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1,
7527                                                     NULL);
7528                         if (rc)
7529                                 return rc;
7530                 }
7531                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7532         }
7533         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7534         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7535         if (rc) {
7536                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7537                            rc);
7538                 return rc;
7539         }
7540         ctx->flags |= BNXT_CTX_FLAG_INITED;
7541         return 0;
7542 }
7543
7544 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7545 {
7546         struct hwrm_func_resource_qcaps_output *resp;
7547         struct hwrm_func_resource_qcaps_input *req;
7548         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7549         int rc;
7550
7551         rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS);
7552         if (rc)
7553                 return rc;
7554
7555         req->fid = cpu_to_le16(0xffff);
7556         resp = hwrm_req_hold(bp, req);
7557         rc = hwrm_req_send_silent(bp, req);
7558         if (rc)
7559                 goto hwrm_func_resc_qcaps_exit;
7560
7561         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7562         if (!all)
7563                 goto hwrm_func_resc_qcaps_exit;
7564
7565         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7566         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7567         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7568         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7569         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7570         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7571         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7572         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7573         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7574         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7575         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7576         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7577         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7578         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7579         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7580         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7581
7582         if (bp->flags & BNXT_FLAG_CHIP_P5) {
7583                 u16 max_msix = le16_to_cpu(resp->max_msix);
7584
7585                 hw_resc->max_nqs = max_msix;
7586                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7587         }
7588
7589         if (BNXT_PF(bp)) {
7590                 struct bnxt_pf_info *pf = &bp->pf;
7591
7592                 pf->vf_resv_strategy =
7593                         le16_to_cpu(resp->vf_reservation_strategy);
7594                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7595                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7596         }
7597 hwrm_func_resc_qcaps_exit:
7598         hwrm_req_drop(bp, req);
7599         return rc;
7600 }
7601
7602 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp)
7603 {
7604         struct hwrm_port_mac_ptp_qcfg_output *resp;
7605         struct hwrm_port_mac_ptp_qcfg_input *req;
7606         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
7607         bool phc_cfg;
7608         u8 flags;
7609         int rc;
7610
7611         if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5_THOR(bp)) {
7612                 rc = -ENODEV;
7613                 goto no_ptp;
7614         }
7615
7616         rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG);
7617         if (rc)
7618                 goto no_ptp;
7619
7620         req->port_id = cpu_to_le16(bp->pf.port_id);
7621         resp = hwrm_req_hold(bp, req);
7622         rc = hwrm_req_send(bp, req);
7623         if (rc)
7624                 goto exit;
7625
7626         flags = resp->flags;
7627         if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) {
7628                 rc = -ENODEV;
7629                 goto exit;
7630         }
7631         if (!ptp) {
7632                 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL);
7633                 if (!ptp) {
7634                         rc = -ENOMEM;
7635                         goto exit;
7636                 }
7637                 ptp->bp = bp;
7638                 bp->ptp_cfg = ptp;
7639         }
7640         if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) {
7641                 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower);
7642                 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper);
7643         } else if (bp->flags & BNXT_FLAG_CHIP_P5) {
7644                 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER;
7645                 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER;
7646         } else {
7647                 rc = -ENODEV;
7648                 goto exit;
7649         }
7650         phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0;
7651         rc = bnxt_ptp_init(bp, phc_cfg);
7652         if (rc)
7653                 netdev_warn(bp->dev, "PTP initialization failed.\n");
7654 exit:
7655         hwrm_req_drop(bp, req);
7656         if (!rc)
7657                 return 0;
7658
7659 no_ptp:
7660         bnxt_ptp_clear(bp);
7661         kfree(ptp);
7662         bp->ptp_cfg = NULL;
7663         return rc;
7664 }
7665
7666 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7667 {
7668         struct hwrm_func_qcaps_output *resp;
7669         struct hwrm_func_qcaps_input *req;
7670         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7671         u32 flags, flags_ext, flags_ext2;
7672         int rc;
7673
7674         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS);
7675         if (rc)
7676                 return rc;
7677
7678         req->fid = cpu_to_le16(0xffff);
7679         resp = hwrm_req_hold(bp, req);
7680         rc = hwrm_req_send(bp, req);
7681         if (rc)
7682                 goto hwrm_func_qcaps_exit;
7683
7684         flags = le32_to_cpu(resp->flags);
7685         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7686                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7687         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7688                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7689         if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7690                 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7691         if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7692                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7693         if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7694                 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7695         if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7696                 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7697         if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7698                 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7699         if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7700                 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7701         if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED)
7702                 bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS;
7703
7704         flags_ext = le32_to_cpu(resp->flags_ext);
7705         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7706                 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7707         if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED))
7708                 bp->fw_cap |= BNXT_FW_CAP_PTP_PPS;
7709         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED)
7710                 bp->fw_cap |= BNXT_FW_CAP_PTP_RTC;
7711         if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT))
7712                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF;
7713         if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED))
7714                 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH;
7715
7716         flags_ext2 = le32_to_cpu(resp->flags_ext2);
7717         if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED)
7718                 bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS;
7719
7720         bp->tx_push_thresh = 0;
7721         if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7722             BNXT_FW_MAJ(bp) > 217)
7723                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7724
7725         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7726         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7727         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7728         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7729         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7730         if (!hw_resc->max_hw_ring_grps)
7731                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7732         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7733         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7734         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7735
7736         if (BNXT_PF(bp)) {
7737                 struct bnxt_pf_info *pf = &bp->pf;
7738
7739                 pf->fw_fid = le16_to_cpu(resp->fid);
7740                 pf->port_id = le16_to_cpu(resp->port_id);
7741                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7742                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7743                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
7744                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7745                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7746                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7747                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7748                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7749                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7750                 bp->flags &= ~BNXT_FLAG_WOL_CAP;
7751                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7752                         bp->flags |= BNXT_FLAG_WOL_CAP;
7753                 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) {
7754                         bp->fw_cap |= BNXT_FW_CAP_PTP;
7755                 } else {
7756                         bnxt_ptp_clear(bp);
7757                         kfree(bp->ptp_cfg);
7758                         bp->ptp_cfg = NULL;
7759                 }
7760         } else {
7761 #ifdef CONFIG_BNXT_SRIOV
7762                 struct bnxt_vf_info *vf = &bp->vf;
7763
7764                 vf->fw_fid = le16_to_cpu(resp->fid);
7765                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7766 #endif
7767         }
7768
7769 hwrm_func_qcaps_exit:
7770         hwrm_req_drop(bp, req);
7771         return rc;
7772 }
7773
7774 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp)
7775 {
7776         struct hwrm_dbg_qcaps_output *resp;
7777         struct hwrm_dbg_qcaps_input *req;
7778         int rc;
7779
7780         bp->fw_dbg_cap = 0;
7781         if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS))
7782                 return;
7783
7784         rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS);
7785         if (rc)
7786                 return;
7787
7788         req->fid = cpu_to_le16(0xffff);
7789         resp = hwrm_req_hold(bp, req);
7790         rc = hwrm_req_send(bp, req);
7791         if (rc)
7792                 goto hwrm_dbg_qcaps_exit;
7793
7794         bp->fw_dbg_cap = le32_to_cpu(resp->flags);
7795
7796 hwrm_dbg_qcaps_exit:
7797         hwrm_req_drop(bp, req);
7798 }
7799
7800 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7801
7802 int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7803 {
7804         int rc;
7805
7806         rc = __bnxt_hwrm_func_qcaps(bp);
7807         if (rc)
7808                 return rc;
7809
7810         bnxt_hwrm_dbg_qcaps(bp);
7811
7812         rc = bnxt_hwrm_queue_qportcfg(bp);
7813         if (rc) {
7814                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7815                 return rc;
7816         }
7817         if (bp->hwrm_spec_code >= 0x10803) {
7818                 rc = bnxt_alloc_ctx_mem(bp);
7819                 if (rc)
7820                         return rc;
7821                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7822                 if (!rc)
7823                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7824         }
7825         return 0;
7826 }
7827
7828 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7829 {
7830         struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7831         struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req;
7832         u32 flags;
7833         int rc;
7834
7835         if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7836                 return 0;
7837
7838         rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS);
7839         if (rc)
7840                 return rc;
7841
7842         resp = hwrm_req_hold(bp, req);
7843         rc = hwrm_req_send(bp, req);
7844         if (rc)
7845                 goto hwrm_cfa_adv_qcaps_exit;
7846
7847         flags = le32_to_cpu(resp->flags);
7848         if (flags &
7849             CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7850                 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7851
7852 hwrm_cfa_adv_qcaps_exit:
7853         hwrm_req_drop(bp, req);
7854         return rc;
7855 }
7856
7857 static int __bnxt_alloc_fw_health(struct bnxt *bp)
7858 {
7859         if (bp->fw_health)
7860                 return 0;
7861
7862         bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
7863         if (!bp->fw_health)
7864                 return -ENOMEM;
7865
7866         mutex_init(&bp->fw_health->lock);
7867         return 0;
7868 }
7869
7870 static int bnxt_alloc_fw_health(struct bnxt *bp)
7871 {
7872         int rc;
7873
7874         if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
7875             !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7876                 return 0;
7877
7878         rc = __bnxt_alloc_fw_health(bp);
7879         if (rc) {
7880                 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
7881                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7882                 return rc;
7883         }
7884
7885         return 0;
7886 }
7887
7888 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
7889 {
7890         writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
7891                                          BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7892                                          BNXT_FW_HEALTH_WIN_MAP_OFF);
7893 }
7894
7895 static void bnxt_inv_fw_health_reg(struct bnxt *bp)
7896 {
7897         struct bnxt_fw_health *fw_health = bp->fw_health;
7898         u32 reg_type;
7899
7900         if (!fw_health)
7901                 return;
7902
7903         reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]);
7904         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
7905                 fw_health->status_reliable = false;
7906
7907         reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]);
7908         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
7909                 fw_health->resets_reliable = false;
7910 }
7911
7912 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
7913 {
7914         void __iomem *hs;
7915         u32 status_loc;
7916         u32 reg_type;
7917         u32 sig;
7918
7919         if (bp->fw_health)
7920                 bp->fw_health->status_reliable = false;
7921
7922         __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
7923         hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
7924
7925         sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
7926         if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
7927                 if (!bp->chip_num) {
7928                         __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE);
7929                         bp->chip_num = readl(bp->bar0 +
7930                                              BNXT_FW_HEALTH_WIN_BASE +
7931                                              BNXT_GRC_REG_CHIP_NUM);
7932                 }
7933                 if (!BNXT_CHIP_P5(bp))
7934                         return;
7935
7936                 status_loc = BNXT_GRC_REG_STATUS_P5 |
7937                              BNXT_FW_HEALTH_REG_TYPE_BAR0;
7938         } else {
7939                 status_loc = readl(hs + offsetof(struct hcomm_status,
7940                                                  fw_status_loc));
7941         }
7942
7943         if (__bnxt_alloc_fw_health(bp)) {
7944                 netdev_warn(bp->dev, "no memory for firmware status checks\n");
7945                 return;
7946         }
7947
7948         bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
7949         reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
7950         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
7951                 __bnxt_map_fw_health_reg(bp, status_loc);
7952                 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
7953                         BNXT_FW_HEALTH_WIN_OFF(status_loc);
7954         }
7955
7956         bp->fw_health->status_reliable = true;
7957 }
7958
7959 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7960 {
7961         struct bnxt_fw_health *fw_health = bp->fw_health;
7962         u32 reg_base = 0xffffffff;
7963         int i;
7964
7965         bp->fw_health->status_reliable = false;
7966         bp->fw_health->resets_reliable = false;
7967         /* Only pre-map the monitoring GRC registers using window 3 */
7968         for (i = 0; i < 4; i++) {
7969                 u32 reg = fw_health->regs[i];
7970
7971                 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7972                         continue;
7973                 if (reg_base == 0xffffffff)
7974                         reg_base = reg & BNXT_GRC_BASE_MASK;
7975                 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7976                         return -ERANGE;
7977                 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
7978         }
7979         bp->fw_health->status_reliable = true;
7980         bp->fw_health->resets_reliable = true;
7981         if (reg_base == 0xffffffff)
7982                 return 0;
7983
7984         __bnxt_map_fw_health_reg(bp, reg_base);
7985         return 0;
7986 }
7987
7988 static void bnxt_remap_fw_health_regs(struct bnxt *bp)
7989 {
7990         if (!bp->fw_health)
7991                 return;
7992
7993         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) {
7994                 bp->fw_health->status_reliable = true;
7995                 bp->fw_health->resets_reliable = true;
7996         } else {
7997                 bnxt_try_map_fw_health_reg(bp);
7998         }
7999 }
8000
8001 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
8002 {
8003         struct bnxt_fw_health *fw_health = bp->fw_health;
8004         struct hwrm_error_recovery_qcfg_output *resp;
8005         struct hwrm_error_recovery_qcfg_input *req;
8006         int rc, i;
8007
8008         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
8009                 return 0;
8010
8011         rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG);
8012         if (rc)
8013                 return rc;
8014
8015         resp = hwrm_req_hold(bp, req);
8016         rc = hwrm_req_send(bp, req);
8017         if (rc)
8018                 goto err_recovery_out;
8019         fw_health->flags = le32_to_cpu(resp->flags);
8020         if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
8021             !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
8022                 rc = -EINVAL;
8023                 goto err_recovery_out;
8024         }
8025         fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
8026         fw_health->master_func_wait_dsecs =
8027                 le32_to_cpu(resp->master_func_wait_period);
8028         fw_health->normal_func_wait_dsecs =
8029                 le32_to_cpu(resp->normal_func_wait_period);
8030         fw_health->post_reset_wait_dsecs =
8031                 le32_to_cpu(resp->master_func_wait_period_after_reset);
8032         fw_health->post_reset_max_wait_dsecs =
8033                 le32_to_cpu(resp->max_bailout_time_after_reset);
8034         fw_health->regs[BNXT_FW_HEALTH_REG] =
8035                 le32_to_cpu(resp->fw_health_status_reg);
8036         fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
8037                 le32_to_cpu(resp->fw_heartbeat_reg);
8038         fw_health->regs[BNXT_FW_RESET_CNT_REG] =
8039                 le32_to_cpu(resp->fw_reset_cnt_reg);
8040         fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
8041                 le32_to_cpu(resp->reset_inprogress_reg);
8042         fw_health->fw_reset_inprog_reg_mask =
8043                 le32_to_cpu(resp->reset_inprogress_reg_mask);
8044         fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
8045         if (fw_health->fw_reset_seq_cnt >= 16) {
8046                 rc = -EINVAL;
8047                 goto err_recovery_out;
8048         }
8049         for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
8050                 fw_health->fw_reset_seq_regs[i] =
8051                         le32_to_cpu(resp->reset_reg[i]);
8052                 fw_health->fw_reset_seq_vals[i] =
8053                         le32_to_cpu(resp->reset_reg_val[i]);
8054                 fw_health->fw_reset_seq_delay_msec[i] =
8055                         resp->delay_after_reset[i];
8056         }
8057 err_recovery_out:
8058         hwrm_req_drop(bp, req);
8059         if (!rc)
8060                 rc = bnxt_map_fw_health_regs(bp);
8061         if (rc)
8062                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
8063         return rc;
8064 }
8065
8066 static int bnxt_hwrm_func_reset(struct bnxt *bp)
8067 {
8068         struct hwrm_func_reset_input *req;
8069         int rc;
8070
8071         rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET);
8072         if (rc)
8073                 return rc;
8074
8075         req->enables = 0;
8076         hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT);
8077         return hwrm_req_send(bp, req);
8078 }
8079
8080 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
8081 {
8082         struct hwrm_nvm_get_dev_info_output nvm_info;
8083
8084         if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
8085                 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
8086                          nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
8087                          nvm_info.nvm_cfg_ver_upd);
8088 }
8089
8090 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
8091 {
8092         struct hwrm_queue_qportcfg_output *resp;
8093         struct hwrm_queue_qportcfg_input *req;
8094         u8 i, j, *qptr;
8095         bool no_rdma;
8096         int rc = 0;
8097
8098         rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG);
8099         if (rc)
8100                 return rc;
8101
8102         resp = hwrm_req_hold(bp, req);
8103         rc = hwrm_req_send(bp, req);
8104         if (rc)
8105                 goto qportcfg_exit;
8106
8107         if (!resp->max_configurable_queues) {
8108                 rc = -EINVAL;
8109                 goto qportcfg_exit;
8110         }
8111         bp->max_tc = resp->max_configurable_queues;
8112         bp->max_lltc = resp->max_configurable_lossless_queues;
8113         if (bp->max_tc > BNXT_MAX_QUEUE)
8114                 bp->max_tc = BNXT_MAX_QUEUE;
8115
8116         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
8117         qptr = &resp->queue_id0;
8118         for (i = 0, j = 0; i < bp->max_tc; i++) {
8119                 bp->q_info[j].queue_id = *qptr;
8120                 bp->q_ids[i] = *qptr++;
8121                 bp->q_info[j].queue_profile = *qptr++;
8122                 bp->tc_to_qidx[j] = j;
8123                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
8124                     (no_rdma && BNXT_PF(bp)))
8125                         j++;
8126         }
8127         bp->max_q = bp->max_tc;
8128         bp->max_tc = max_t(u8, j, 1);
8129
8130         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
8131                 bp->max_tc = 1;
8132
8133         if (bp->max_lltc > bp->max_tc)
8134                 bp->max_lltc = bp->max_tc;
8135
8136 qportcfg_exit:
8137         hwrm_req_drop(bp, req);
8138         return rc;
8139 }
8140
8141 static int bnxt_hwrm_poll(struct bnxt *bp)
8142 {
8143         struct hwrm_ver_get_input *req;
8144         int rc;
8145
8146         rc = hwrm_req_init(bp, req, HWRM_VER_GET);
8147         if (rc)
8148                 return rc;
8149
8150         req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
8151         req->hwrm_intf_min = HWRM_VERSION_MINOR;
8152         req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
8153
8154         hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT);
8155         rc = hwrm_req_send(bp, req);
8156         return rc;
8157 }
8158
8159 static int bnxt_hwrm_ver_get(struct bnxt *bp)
8160 {
8161         struct hwrm_ver_get_output *resp;
8162         struct hwrm_ver_get_input *req;
8163         u16 fw_maj, fw_min, fw_bld, fw_rsv;
8164         u32 dev_caps_cfg, hwrm_ver;
8165         int rc, len;
8166
8167         rc = hwrm_req_init(bp, req, HWRM_VER_GET);
8168         if (rc)
8169                 return rc;
8170
8171         hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
8172         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
8173         req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
8174         req->hwrm_intf_min = HWRM_VERSION_MINOR;
8175         req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
8176
8177         resp = hwrm_req_hold(bp, req);
8178         rc = hwrm_req_send(bp, req);
8179         if (rc)
8180                 goto hwrm_ver_get_exit;
8181
8182         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
8183
8184         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
8185                              resp->hwrm_intf_min_8b << 8 |
8186                              resp->hwrm_intf_upd_8b;
8187         if (resp->hwrm_intf_maj_8b < 1) {
8188                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
8189                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
8190                             resp->hwrm_intf_upd_8b);
8191                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
8192         }
8193
8194         hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
8195                         HWRM_VERSION_UPDATE;
8196
8197         if (bp->hwrm_spec_code > hwrm_ver)
8198                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
8199                          HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
8200                          HWRM_VERSION_UPDATE);
8201         else
8202                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
8203                          resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
8204                          resp->hwrm_intf_upd_8b);
8205
8206         fw_maj = le16_to_cpu(resp->hwrm_fw_major);
8207         if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
8208                 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
8209                 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
8210                 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
8211                 len = FW_VER_STR_LEN;
8212         } else {
8213                 fw_maj = resp->hwrm_fw_maj_8b;
8214                 fw_min = resp->hwrm_fw_min_8b;
8215                 fw_bld = resp->hwrm_fw_bld_8b;
8216                 fw_rsv = resp->hwrm_fw_rsvd_8b;
8217                 len = BC_HWRM_STR_LEN;
8218         }
8219         bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
8220         snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
8221                  fw_rsv);
8222
8223         if (strlen(resp->active_pkg_name)) {
8224                 int fw_ver_len = strlen(bp->fw_ver_str);
8225
8226                 snprintf(bp->fw_ver_str + fw_ver_len,
8227                          FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
8228                          resp->active_pkg_name);
8229                 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
8230         }
8231
8232         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
8233         if (!bp->hwrm_cmd_timeout)
8234                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
8235         bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000;
8236         if (!bp->hwrm_cmd_max_timeout)
8237                 bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT;
8238         else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT)
8239                 netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n",
8240                             bp->hwrm_cmd_max_timeout / 1000);
8241
8242         if (resp->hwrm_intf_maj_8b >= 1) {
8243                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
8244                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
8245         }
8246         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
8247                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
8248
8249         bp->chip_num = le16_to_cpu(resp->chip_num);
8250         bp->chip_rev = resp->chip_rev;
8251         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
8252             !resp->chip_metal)
8253                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
8254
8255         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
8256         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
8257             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
8258                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
8259
8260         if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
8261                 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
8262
8263         if (dev_caps_cfg &
8264             VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
8265                 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
8266
8267         if (dev_caps_cfg &
8268             VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
8269                 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
8270
8271         if (dev_caps_cfg &
8272             VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
8273                 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
8274
8275 hwrm_ver_get_exit:
8276         hwrm_req_drop(bp, req);
8277         return rc;
8278 }
8279
8280 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
8281 {
8282         struct hwrm_fw_set_time_input *req;
8283         struct tm tm;
8284         time64_t now = ktime_get_real_seconds();
8285         int rc;
8286
8287         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
8288             bp->hwrm_spec_code < 0x10400)
8289                 return -EOPNOTSUPP;
8290
8291         time64_to_tm(now, 0, &tm);
8292         rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME);
8293         if (rc)
8294                 return rc;
8295
8296         req->year = cpu_to_le16(1900 + tm.tm_year);
8297         req->month = 1 + tm.tm_mon;
8298         req->day = tm.tm_mday;
8299         req->hour = tm.tm_hour;
8300         req->minute = tm.tm_min;
8301         req->second = tm.tm_sec;
8302         return hwrm_req_send(bp, req);
8303 }
8304
8305 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
8306 {
8307         u64 sw_tmp;
8308
8309         hw &= mask;
8310         sw_tmp = (*sw & ~mask) | hw;
8311         if (hw < (*sw & mask))
8312                 sw_tmp += mask + 1;
8313         WRITE_ONCE(*sw, sw_tmp);
8314 }
8315
8316 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
8317                                     int count, bool ignore_zero)
8318 {
8319         int i;
8320
8321         for (i = 0; i < count; i++) {
8322                 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
8323
8324                 if (ignore_zero && !hw)
8325                         continue;
8326
8327                 if (masks[i] == -1ULL)
8328                         sw_stats[i] = hw;
8329                 else
8330                         bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
8331         }
8332 }
8333
8334 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
8335 {
8336         if (!stats->hw_stats)
8337                 return;
8338
8339         __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
8340                                 stats->hw_masks, stats->len / 8, false);
8341 }
8342
8343 static void bnxt_accumulate_all_stats(struct bnxt *bp)
8344 {
8345         struct bnxt_stats_mem *ring0_stats;
8346         bool ignore_zero = false;
8347         int i;
8348
8349         /* Chip bug.  Counter intermittently becomes 0. */
8350         if (bp->flags & BNXT_FLAG_CHIP_P5)
8351                 ignore_zero = true;
8352
8353         for (i = 0; i < bp->cp_nr_rings; i++) {
8354                 struct bnxt_napi *bnapi = bp->bnapi[i];
8355                 struct bnxt_cp_ring_info *cpr;
8356                 struct bnxt_stats_mem *stats;
8357
8358                 cpr = &bnapi->cp_ring;
8359                 stats = &cpr->stats;
8360                 if (!i)
8361                         ring0_stats = stats;
8362                 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
8363                                         ring0_stats->hw_masks,
8364                                         ring0_stats->len / 8, ignore_zero);
8365         }
8366         if (bp->flags & BNXT_FLAG_PORT_STATS) {
8367                 struct bnxt_stats_mem *stats = &bp->port_stats;
8368                 __le64 *hw_stats = stats->hw_stats;
8369                 u64 *sw_stats = stats->sw_stats;
8370                 u64 *masks = stats->hw_masks;
8371                 int cnt;
8372
8373                 cnt = sizeof(struct rx_port_stats) / 8;
8374                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
8375
8376                 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8377                 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8378                 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8379                 cnt = sizeof(struct tx_port_stats) / 8;
8380                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
8381         }
8382         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
8383                 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
8384                 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
8385         }
8386 }
8387
8388 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
8389 {
8390         struct hwrm_port_qstats_input *req;
8391         struct bnxt_pf_info *pf = &bp->pf;
8392         int rc;
8393
8394         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
8395                 return 0;
8396
8397         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
8398                 return -EOPNOTSUPP;
8399
8400         rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS);
8401         if (rc)
8402                 return rc;
8403
8404         req->flags = flags;
8405         req->port_id = cpu_to_le16(pf->port_id);
8406         req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
8407                                             BNXT_TX_PORT_STATS_BYTE_OFFSET);
8408         req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
8409         return hwrm_req_send(bp, req);
8410 }
8411
8412 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
8413 {
8414         struct hwrm_queue_pri2cos_qcfg_output *resp_qc;
8415         struct hwrm_queue_pri2cos_qcfg_input *req_qc;
8416         struct hwrm_port_qstats_ext_output *resp_qs;
8417         struct hwrm_port_qstats_ext_input *req_qs;
8418         struct bnxt_pf_info *pf = &bp->pf;
8419         u32 tx_stat_size;
8420         int rc;
8421
8422         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
8423                 return 0;
8424
8425         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
8426                 return -EOPNOTSUPP;
8427
8428         rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT);
8429         if (rc)
8430                 return rc;
8431
8432         req_qs->flags = flags;
8433         req_qs->port_id = cpu_to_le16(pf->port_id);
8434         req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
8435         req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
8436         tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
8437                        sizeof(struct tx_port_stats_ext) : 0;
8438         req_qs->tx_stat_size = cpu_to_le16(tx_stat_size);
8439         req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
8440         resp_qs = hwrm_req_hold(bp, req_qs);
8441         rc = hwrm_req_send(bp, req_qs);
8442         if (!rc) {
8443                 bp->fw_rx_stats_ext_size =
8444                         le16_to_cpu(resp_qs->rx_stat_size) / 8;
8445                 if (BNXT_FW_MAJ(bp) < 220 &&
8446                     bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY)
8447                         bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY;
8448
8449                 bp->fw_tx_stats_ext_size = tx_stat_size ?
8450                         le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0;
8451         } else {
8452                 bp->fw_rx_stats_ext_size = 0;
8453                 bp->fw_tx_stats_ext_size = 0;
8454         }
8455         hwrm_req_drop(bp, req_qs);
8456
8457         if (flags)
8458                 return rc;
8459
8460         if (bp->fw_tx_stats_ext_size <=
8461             offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
8462                 bp->pri2cos_valid = 0;
8463                 return rc;
8464         }
8465
8466         rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG);
8467         if (rc)
8468                 return rc;
8469
8470         req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
8471
8472         resp_qc = hwrm_req_hold(bp, req_qc);
8473         rc = hwrm_req_send(bp, req_qc);
8474         if (!rc) {
8475                 u8 *pri2cos;
8476                 int i, j;
8477
8478                 pri2cos = &resp_qc->pri0_cos_queue_id;
8479                 for (i = 0; i < 8; i++) {
8480                         u8 queue_id = pri2cos[i];
8481                         u8 queue_idx;
8482
8483                         /* Per port queue IDs start from 0, 10, 20, etc */
8484                         queue_idx = queue_id % 10;
8485                         if (queue_idx > BNXT_MAX_QUEUE) {
8486                                 bp->pri2cos_valid = false;
8487                                 hwrm_req_drop(bp, req_qc);
8488                                 return rc;
8489                         }
8490                         for (j = 0; j < bp->max_q; j++) {
8491                                 if (bp->q_ids[j] == queue_id)
8492                                         bp->pri2cos_idx[i] = queue_idx;
8493                         }
8494                 }
8495                 bp->pri2cos_valid = true;
8496         }
8497         hwrm_req_drop(bp, req_qc);
8498
8499         return rc;
8500 }
8501
8502 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
8503 {
8504         bnxt_hwrm_tunnel_dst_port_free(bp,
8505                 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
8506         bnxt_hwrm_tunnel_dst_port_free(bp,
8507                 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
8508 }
8509
8510 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
8511 {
8512         int rc, i;
8513         u32 tpa_flags = 0;
8514
8515         if (set_tpa)
8516                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
8517         else if (BNXT_NO_FW_ACCESS(bp))
8518                 return 0;
8519         for (i = 0; i < bp->nr_vnics; i++) {
8520                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
8521                 if (rc) {
8522                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
8523                                    i, rc);
8524                         return rc;
8525                 }
8526         }
8527         return 0;
8528 }
8529
8530 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
8531 {
8532         int i;
8533
8534         for (i = 0; i < bp->nr_vnics; i++)
8535                 bnxt_hwrm_vnic_set_rss(bp, i, false);
8536 }
8537
8538 static void bnxt_clear_vnic(struct bnxt *bp)
8539 {
8540         if (!bp->vnic_info)
8541                 return;
8542
8543         bnxt_hwrm_clear_vnic_filter(bp);
8544         if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
8545                 /* clear all RSS setting before free vnic ctx */
8546                 bnxt_hwrm_clear_vnic_rss(bp);
8547                 bnxt_hwrm_vnic_ctx_free(bp);
8548         }
8549         /* before free the vnic, undo the vnic tpa settings */
8550         if (bp->flags & BNXT_FLAG_TPA)
8551                 bnxt_set_tpa(bp, false);
8552         bnxt_hwrm_vnic_free(bp);
8553         if (bp->flags & BNXT_FLAG_CHIP_P5)
8554                 bnxt_hwrm_vnic_ctx_free(bp);
8555 }
8556
8557 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
8558                                     bool irq_re_init)
8559 {
8560         bnxt_clear_vnic(bp);
8561         bnxt_hwrm_ring_free(bp, close_path);
8562         bnxt_hwrm_ring_grp_free(bp);
8563         if (irq_re_init) {
8564                 bnxt_hwrm_stat_ctx_free(bp);
8565                 bnxt_hwrm_free_tunnel_ports(bp);
8566         }
8567 }
8568
8569 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
8570 {
8571         struct hwrm_func_cfg_input *req;
8572         u8 evb_mode;
8573         int rc;
8574
8575         if (br_mode == BRIDGE_MODE_VEB)
8576                 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
8577         else if (br_mode == BRIDGE_MODE_VEPA)
8578                 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
8579         else
8580                 return -EINVAL;
8581
8582         rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
8583         if (rc)
8584                 return rc;
8585
8586         req->fid = cpu_to_le16(0xffff);
8587         req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
8588         req->evb_mode = evb_mode;
8589         return hwrm_req_send(bp, req);
8590 }
8591
8592 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
8593 {
8594         struct hwrm_func_cfg_input *req;
8595         int rc;
8596
8597         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
8598                 return 0;
8599
8600         rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
8601         if (rc)
8602                 return rc;
8603
8604         req->fid = cpu_to_le16(0xffff);
8605         req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
8606         req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
8607         if (size == 128)
8608                 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
8609
8610         return hwrm_req_send(bp, req);
8611 }
8612
8613 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8614 {
8615         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
8616         int rc;
8617
8618         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
8619                 goto skip_rss_ctx;
8620
8621         /* allocate context for vnic */
8622         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
8623         if (rc) {
8624                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8625                            vnic_id, rc);
8626                 goto vnic_setup_err;
8627         }
8628         bp->rsscos_nr_ctxs++;
8629
8630         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8631                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
8632                 if (rc) {
8633                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
8634                                    vnic_id, rc);
8635                         goto vnic_setup_err;
8636                 }
8637                 bp->rsscos_nr_ctxs++;
8638         }
8639
8640 skip_rss_ctx:
8641         /* configure default vnic, ring grp */
8642         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8643         if (rc) {
8644                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8645                            vnic_id, rc);
8646                 goto vnic_setup_err;
8647         }
8648
8649         /* Enable RSS hashing on vnic */
8650         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
8651         if (rc) {
8652                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
8653                            vnic_id, rc);
8654                 goto vnic_setup_err;
8655         }
8656
8657         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8658                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8659                 if (rc) {
8660                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8661                                    vnic_id, rc);
8662                 }
8663         }
8664
8665 vnic_setup_err:
8666         return rc;
8667 }
8668
8669 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
8670 {
8671         int rc, i, nr_ctxs;
8672
8673         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
8674         for (i = 0; i < nr_ctxs; i++) {
8675                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
8676                 if (rc) {
8677                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
8678                                    vnic_id, i, rc);
8679                         break;
8680                 }
8681                 bp->rsscos_nr_ctxs++;
8682         }
8683         if (i < nr_ctxs)
8684                 return -ENOMEM;
8685
8686         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
8687         if (rc) {
8688                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
8689                            vnic_id, rc);
8690                 return rc;
8691         }
8692         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8693         if (rc) {
8694                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8695                            vnic_id, rc);
8696                 return rc;
8697         }
8698         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8699                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8700                 if (rc) {
8701                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8702                                    vnic_id, rc);
8703                 }
8704         }
8705         return rc;
8706 }
8707
8708 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8709 {
8710         if (bp->flags & BNXT_FLAG_CHIP_P5)
8711                 return __bnxt_setup_vnic_p5(bp, vnic_id);
8712         else
8713                 return __bnxt_setup_vnic(bp, vnic_id);
8714 }
8715
8716 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8717 {
8718 #ifdef CONFIG_RFS_ACCEL
8719         int i, rc = 0;
8720
8721         if (bp->flags & BNXT_FLAG_CHIP_P5)
8722                 return 0;
8723
8724         for (i = 0; i < bp->rx_nr_rings; i++) {
8725                 struct bnxt_vnic_info *vnic;
8726                 u16 vnic_id = i + 1;
8727                 u16 ring_id = i;
8728
8729                 if (vnic_id >= bp->nr_vnics)
8730                         break;
8731
8732                 vnic = &bp->vnic_info[vnic_id];
8733                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
8734                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8735                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8736                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8737                 if (rc) {
8738                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8739                                    vnic_id, rc);
8740                         break;
8741                 }
8742                 rc = bnxt_setup_vnic(bp, vnic_id);
8743                 if (rc)
8744                         break;
8745         }
8746         return rc;
8747 #else
8748         return 0;
8749 #endif
8750 }
8751
8752 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */
8753 static bool bnxt_promisc_ok(struct bnxt *bp)
8754 {
8755 #ifdef CONFIG_BNXT_SRIOV
8756         if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf))
8757                 return false;
8758 #endif
8759         return true;
8760 }
8761
8762 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8763 {
8764         unsigned int rc = 0;
8765
8766         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8767         if (rc) {
8768                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8769                            rc);
8770                 return rc;
8771         }
8772
8773         rc = bnxt_hwrm_vnic_cfg(bp, 1);
8774         if (rc) {
8775                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8776                            rc);
8777                 return rc;
8778         }
8779         return rc;
8780 }
8781
8782 static int bnxt_cfg_rx_mode(struct bnxt *);
8783 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8784
8785 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8786 {
8787         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8788         int rc = 0;
8789         unsigned int rx_nr_rings = bp->rx_nr_rings;
8790
8791         if (irq_re_init) {
8792                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
8793                 if (rc) {
8794                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8795                                    rc);
8796                         goto err_out;
8797                 }
8798         }
8799
8800         rc = bnxt_hwrm_ring_alloc(bp);
8801         if (rc) {
8802                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8803                 goto err_out;
8804         }
8805
8806         rc = bnxt_hwrm_ring_grp_alloc(bp);
8807         if (rc) {
8808                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8809                 goto err_out;
8810         }
8811
8812         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8813                 rx_nr_rings--;
8814
8815         /* default vnic 0 */
8816         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8817         if (rc) {
8818                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8819                 goto err_out;
8820         }
8821
8822         if (BNXT_VF(bp))
8823                 bnxt_hwrm_func_qcfg(bp);
8824
8825         rc = bnxt_setup_vnic(bp, 0);
8826         if (rc)
8827                 goto err_out;
8828         if (bp->fw_cap & BNXT_FW_CAP_RSS_HASH_TYPE_DELTA)
8829                 bnxt_hwrm_update_rss_hash_cfg(bp);
8830
8831         if (bp->flags & BNXT_FLAG_RFS) {
8832                 rc = bnxt_alloc_rfs_vnics(bp);
8833                 if (rc)
8834                         goto err_out;
8835         }
8836
8837         if (bp->flags & BNXT_FLAG_TPA) {
8838                 rc = bnxt_set_tpa(bp, true);
8839                 if (rc)
8840                         goto err_out;
8841         }
8842
8843         if (BNXT_VF(bp))
8844                 bnxt_update_vf_mac(bp);
8845
8846         /* Filter for default vnic 0 */
8847         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8848         if (rc) {
8849                 if (BNXT_VF(bp) && rc == -ENODEV)
8850                         netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n");
8851                 else
8852                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8853                 goto err_out;
8854         }
8855         vnic->uc_filter_count = 1;
8856
8857         vnic->rx_mask = 0;
8858         if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state))
8859                 goto skip_rx_mask;
8860
8861         if (bp->dev->flags & IFF_BROADCAST)
8862                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8863
8864         if (bp->dev->flags & IFF_PROMISC)
8865                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8866
8867         if (bp->dev->flags & IFF_ALLMULTI) {
8868                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8869                 vnic->mc_list_count = 0;
8870         } else if (bp->dev->flags & IFF_MULTICAST) {
8871                 u32 mask = 0;
8872
8873                 bnxt_mc_list_updated(bp, &mask);
8874                 vnic->rx_mask |= mask;
8875         }
8876
8877         rc = bnxt_cfg_rx_mode(bp);
8878         if (rc)
8879                 goto err_out;
8880
8881 skip_rx_mask:
8882         rc = bnxt_hwrm_set_coal(bp);
8883         if (rc)
8884                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8885                                 rc);
8886
8887         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8888                 rc = bnxt_setup_nitroa0_vnic(bp);
8889                 if (rc)
8890                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8891                                    rc);
8892         }
8893
8894         if (BNXT_VF(bp)) {
8895                 bnxt_hwrm_func_qcfg(bp);
8896                 netdev_update_features(bp->dev);
8897         }
8898
8899         return 0;
8900
8901 err_out:
8902         bnxt_hwrm_resource_free(bp, 0, true);
8903
8904         return rc;
8905 }
8906
8907 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8908 {
8909         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8910         return 0;
8911 }
8912
8913 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8914 {
8915         bnxt_init_cp_rings(bp);
8916         bnxt_init_rx_rings(bp);
8917         bnxt_init_tx_rings(bp);
8918         bnxt_init_ring_grps(bp, irq_re_init);
8919         bnxt_init_vnics(bp);
8920
8921         return bnxt_init_chip(bp, irq_re_init);
8922 }
8923
8924 static int bnxt_set_real_num_queues(struct bnxt *bp)
8925 {
8926         int rc;
8927         struct net_device *dev = bp->dev;
8928
8929         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8930                                           bp->tx_nr_rings_xdp);
8931         if (rc)
8932                 return rc;
8933
8934         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8935         if (rc)
8936                 return rc;
8937
8938 #ifdef CONFIG_RFS_ACCEL
8939         if (bp->flags & BNXT_FLAG_RFS)
8940                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8941 #endif
8942
8943         return rc;
8944 }
8945
8946 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8947                            bool shared)
8948 {
8949         int _rx = *rx, _tx = *tx;
8950
8951         if (shared) {
8952                 *rx = min_t(int, _rx, max);
8953                 *tx = min_t(int, _tx, max);
8954         } else {
8955                 if (max < 2)
8956                         return -ENOMEM;
8957
8958                 while (_rx + _tx > max) {
8959                         if (_rx > _tx && _rx > 1)
8960                                 _rx--;
8961                         else if (_tx > 1)
8962                                 _tx--;
8963                 }
8964                 *rx = _rx;
8965                 *tx = _tx;
8966         }
8967         return 0;
8968 }
8969
8970 static void bnxt_setup_msix(struct bnxt *bp)
8971 {
8972         const int len = sizeof(bp->irq_tbl[0].name);
8973         struct net_device *dev = bp->dev;
8974         int tcs, i;
8975
8976         tcs = netdev_get_num_tc(dev);
8977         if (tcs) {
8978                 int i, off, count;
8979
8980                 for (i = 0; i < tcs; i++) {
8981                         count = bp->tx_nr_rings_per_tc;
8982                         off = i * count;
8983                         netdev_set_tc_queue(dev, i, count, off);
8984                 }
8985         }
8986
8987         for (i = 0; i < bp->cp_nr_rings; i++) {
8988                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8989                 char *attr;
8990
8991                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8992                         attr = "TxRx";
8993                 else if (i < bp->rx_nr_rings)
8994                         attr = "rx";
8995                 else
8996                         attr = "tx";
8997
8998                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8999                          attr, i);
9000                 bp->irq_tbl[map_idx].handler = bnxt_msix;
9001         }
9002 }
9003
9004 static void bnxt_setup_inta(struct bnxt *bp)
9005 {
9006         const int len = sizeof(bp->irq_tbl[0].name);
9007
9008         if (netdev_get_num_tc(bp->dev))
9009                 netdev_reset_tc(bp->dev);
9010
9011         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
9012                  0);
9013         bp->irq_tbl[0].handler = bnxt_inta;
9014 }
9015
9016 static int bnxt_init_int_mode(struct bnxt *bp);
9017
9018 static int bnxt_setup_int_mode(struct bnxt *bp)
9019 {
9020         int rc;
9021
9022         if (!bp->irq_tbl) {
9023                 rc = bnxt_init_int_mode(bp);
9024                 if (rc || !bp->irq_tbl)
9025                         return rc ?: -ENODEV;
9026         }
9027
9028         if (bp->flags & BNXT_FLAG_USING_MSIX)
9029                 bnxt_setup_msix(bp);
9030         else
9031                 bnxt_setup_inta(bp);
9032
9033         rc = bnxt_set_real_num_queues(bp);
9034         return rc;
9035 }
9036
9037 #ifdef CONFIG_RFS_ACCEL
9038 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
9039 {
9040         return bp->hw_resc.max_rsscos_ctxs;
9041 }
9042
9043 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
9044 {
9045         return bp->hw_resc.max_vnics;
9046 }
9047 #endif
9048
9049 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
9050 {
9051         return bp->hw_resc.max_stat_ctxs;
9052 }
9053
9054 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
9055 {
9056         return bp->hw_resc.max_cp_rings;
9057 }
9058
9059 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
9060 {
9061         unsigned int cp = bp->hw_resc.max_cp_rings;
9062
9063         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9064                 cp -= bnxt_get_ulp_msix_num(bp);
9065
9066         return cp;
9067 }
9068
9069 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
9070 {
9071         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9072
9073         if (bp->flags & BNXT_FLAG_CHIP_P5)
9074                 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
9075
9076         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
9077 }
9078
9079 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
9080 {
9081         bp->hw_resc.max_irqs = max_irqs;
9082 }
9083
9084 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
9085 {
9086         unsigned int cp;
9087
9088         cp = bnxt_get_max_func_cp_rings_for_en(bp);
9089         if (bp->flags & BNXT_FLAG_CHIP_P5)
9090                 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
9091         else
9092                 return cp - bp->cp_nr_rings;
9093 }
9094
9095 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
9096 {
9097         return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
9098 }
9099
9100 int bnxt_get_avail_msix(struct bnxt *bp, int num)
9101 {
9102         int max_cp = bnxt_get_max_func_cp_rings(bp);
9103         int max_irq = bnxt_get_max_func_irqs(bp);
9104         int total_req = bp->cp_nr_rings + num;
9105         int max_idx, avail_msix;
9106
9107         max_idx = bp->total_irqs;
9108         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9109                 max_idx = min_t(int, bp->total_irqs, max_cp);
9110         avail_msix = max_idx - bp->cp_nr_rings;
9111         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
9112                 return avail_msix;
9113
9114         if (max_irq < total_req) {
9115                 num = max_irq - bp->cp_nr_rings;
9116                 if (num <= 0)
9117                         return 0;
9118         }
9119         return num;
9120 }
9121
9122 static int bnxt_get_num_msix(struct bnxt *bp)
9123 {
9124         if (!BNXT_NEW_RM(bp))
9125                 return bnxt_get_max_func_irqs(bp);
9126
9127         return bnxt_nq_rings_in_use(bp);
9128 }
9129
9130 static int bnxt_init_msix(struct bnxt *bp)
9131 {
9132         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
9133         struct msix_entry *msix_ent;
9134
9135         total_vecs = bnxt_get_num_msix(bp);
9136         max = bnxt_get_max_func_irqs(bp);
9137         if (total_vecs > max)
9138                 total_vecs = max;
9139
9140         if (!total_vecs)
9141                 return 0;
9142
9143         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
9144         if (!msix_ent)
9145                 return -ENOMEM;
9146
9147         for (i = 0; i < total_vecs; i++) {
9148                 msix_ent[i].entry = i;
9149                 msix_ent[i].vector = 0;
9150         }
9151
9152         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
9153                 min = 2;
9154
9155         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
9156         ulp_msix = bnxt_get_ulp_msix_num(bp);
9157         if (total_vecs < 0 || total_vecs < ulp_msix) {
9158                 rc = -ENODEV;
9159                 goto msix_setup_exit;
9160         }
9161
9162         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
9163         if (bp->irq_tbl) {
9164                 for (i = 0; i < total_vecs; i++)
9165                         bp->irq_tbl[i].vector = msix_ent[i].vector;
9166
9167                 bp->total_irqs = total_vecs;
9168                 /* Trim rings based upon num of vectors allocated */
9169                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
9170                                      total_vecs - ulp_msix, min == 1);
9171                 if (rc)
9172                         goto msix_setup_exit;
9173
9174                 bp->cp_nr_rings = (min == 1) ?
9175                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
9176                                   bp->tx_nr_rings + bp->rx_nr_rings;
9177
9178         } else {
9179                 rc = -ENOMEM;
9180                 goto msix_setup_exit;
9181         }
9182         bp->flags |= BNXT_FLAG_USING_MSIX;
9183         kfree(msix_ent);
9184         return 0;
9185
9186 msix_setup_exit:
9187         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
9188         kfree(bp->irq_tbl);
9189         bp->irq_tbl = NULL;
9190         pci_disable_msix(bp->pdev);
9191         kfree(msix_ent);
9192         return rc;
9193 }
9194
9195 static int bnxt_init_inta(struct bnxt *bp)
9196 {
9197         bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL);
9198         if (!bp->irq_tbl)
9199                 return -ENOMEM;
9200
9201         bp->total_irqs = 1;
9202         bp->rx_nr_rings = 1;
9203         bp->tx_nr_rings = 1;
9204         bp->cp_nr_rings = 1;
9205         bp->flags |= BNXT_FLAG_SHARED_RINGS;
9206         bp->irq_tbl[0].vector = bp->pdev->irq;
9207         return 0;
9208 }
9209
9210 static int bnxt_init_int_mode(struct bnxt *bp)
9211 {
9212         int rc = -ENODEV;
9213
9214         if (bp->flags & BNXT_FLAG_MSIX_CAP)
9215                 rc = bnxt_init_msix(bp);
9216
9217         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
9218                 /* fallback to INTA */
9219                 rc = bnxt_init_inta(bp);
9220         }
9221         return rc;
9222 }
9223
9224 static void bnxt_clear_int_mode(struct bnxt *bp)
9225 {
9226         if (bp->flags & BNXT_FLAG_USING_MSIX)
9227                 pci_disable_msix(bp->pdev);
9228
9229         kfree(bp->irq_tbl);
9230         bp->irq_tbl = NULL;
9231         bp->flags &= ~BNXT_FLAG_USING_MSIX;
9232 }
9233
9234 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
9235 {
9236         int tcs = netdev_get_num_tc(bp->dev);
9237         bool irq_cleared = false;
9238         int rc;
9239
9240         if (!bnxt_need_reserve_rings(bp))
9241                 return 0;
9242
9243         if (irq_re_init && BNXT_NEW_RM(bp) &&
9244             bnxt_get_num_msix(bp) != bp->total_irqs) {
9245                 bnxt_ulp_irq_stop(bp);
9246                 bnxt_clear_int_mode(bp);
9247                 irq_cleared = true;
9248         }
9249         rc = __bnxt_reserve_rings(bp);
9250         if (irq_cleared) {
9251                 if (!rc)
9252                         rc = bnxt_init_int_mode(bp);
9253                 bnxt_ulp_irq_restart(bp, rc);
9254         }
9255         if (rc) {
9256                 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
9257                 return rc;
9258         }
9259         if (tcs && (bp->tx_nr_rings_per_tc * tcs !=
9260                     bp->tx_nr_rings - bp->tx_nr_rings_xdp)) {
9261                 netdev_err(bp->dev, "tx ring reservation failure\n");
9262                 netdev_reset_tc(bp->dev);
9263                 if (bp->tx_nr_rings_xdp)
9264                         bp->tx_nr_rings_per_tc = bp->tx_nr_rings_xdp;
9265                 else
9266                         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
9267                 return -ENOMEM;
9268         }
9269         return 0;
9270 }
9271
9272 static void bnxt_free_irq(struct bnxt *bp)
9273 {
9274         struct bnxt_irq *irq;
9275         int i;
9276
9277 #ifdef CONFIG_RFS_ACCEL
9278         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
9279         bp->dev->rx_cpu_rmap = NULL;
9280 #endif
9281         if (!bp->irq_tbl || !bp->bnapi)
9282                 return;
9283
9284         for (i = 0; i < bp->cp_nr_rings; i++) {
9285                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
9286
9287                 irq = &bp->irq_tbl[map_idx];
9288                 if (irq->requested) {
9289                         if (irq->have_cpumask) {
9290                                 irq_set_affinity_hint(irq->vector, NULL);
9291                                 free_cpumask_var(irq->cpu_mask);
9292                                 irq->have_cpumask = 0;
9293                         }
9294                         free_irq(irq->vector, bp->bnapi[i]);
9295                 }
9296
9297                 irq->requested = 0;
9298         }
9299 }
9300
9301 static int bnxt_request_irq(struct bnxt *bp)
9302 {
9303         int i, j, rc = 0;
9304         unsigned long flags = 0;
9305 #ifdef CONFIG_RFS_ACCEL
9306         struct cpu_rmap *rmap;
9307 #endif
9308
9309         rc = bnxt_setup_int_mode(bp);
9310         if (rc) {
9311                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
9312                            rc);
9313                 return rc;
9314         }
9315 #ifdef CONFIG_RFS_ACCEL
9316         rmap = bp->dev->rx_cpu_rmap;
9317 #endif
9318         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
9319                 flags = IRQF_SHARED;
9320
9321         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
9322                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
9323                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
9324
9325 #ifdef CONFIG_RFS_ACCEL
9326                 if (rmap && bp->bnapi[i]->rx_ring) {
9327                         rc = irq_cpu_rmap_add(rmap, irq->vector);
9328                         if (rc)
9329                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
9330                                             j);
9331                         j++;
9332                 }
9333 #endif
9334                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
9335                                  bp->bnapi[i]);
9336                 if (rc)
9337                         break;
9338
9339                 irq->requested = 1;
9340
9341                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
9342                         int numa_node = dev_to_node(&bp->pdev->dev);
9343
9344                         irq->have_cpumask = 1;
9345                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
9346                                         irq->cpu_mask);
9347                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
9348                         if (rc) {
9349                                 netdev_warn(bp->dev,
9350                                             "Set affinity failed, IRQ = %d\n",
9351                                             irq->vector);
9352                                 break;
9353                         }
9354                 }
9355         }
9356         return rc;
9357 }
9358
9359 static void bnxt_del_napi(struct bnxt *bp)
9360 {
9361         int i;
9362
9363         if (!bp->bnapi)
9364                 return;
9365
9366         for (i = 0; i < bp->cp_nr_rings; i++) {
9367                 struct bnxt_napi *bnapi = bp->bnapi[i];
9368
9369                 __netif_napi_del(&bnapi->napi);
9370         }
9371         /* We called __netif_napi_del(), we need
9372          * to respect an RCU grace period before freeing napi structures.
9373          */
9374         synchronize_net();
9375 }
9376
9377 static void bnxt_init_napi(struct bnxt *bp)
9378 {
9379         int i;
9380         unsigned int cp_nr_rings = bp->cp_nr_rings;
9381         struct bnxt_napi *bnapi;
9382
9383         if (bp->flags & BNXT_FLAG_USING_MSIX) {
9384                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
9385
9386                 if (bp->flags & BNXT_FLAG_CHIP_P5)
9387                         poll_fn = bnxt_poll_p5;
9388                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
9389                         cp_nr_rings--;
9390                 for (i = 0; i < cp_nr_rings; i++) {
9391                         bnapi = bp->bnapi[i];
9392                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn);
9393                 }
9394                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
9395                         bnapi = bp->bnapi[cp_nr_rings];
9396                         netif_napi_add(bp->dev, &bnapi->napi,
9397                                        bnxt_poll_nitroa0);
9398                 }
9399         } else {
9400                 bnapi = bp->bnapi[0];
9401                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll);
9402         }
9403 }
9404
9405 static void bnxt_disable_napi(struct bnxt *bp)
9406 {
9407         int i;
9408
9409         if (!bp->bnapi ||
9410             test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state))
9411                 return;
9412
9413         for (i = 0; i < bp->cp_nr_rings; i++) {
9414                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
9415
9416                 napi_disable(&bp->bnapi[i]->napi);
9417                 if (bp->bnapi[i]->rx_ring)
9418                         cancel_work_sync(&cpr->dim.work);
9419         }
9420 }
9421
9422 static void bnxt_enable_napi(struct bnxt *bp)
9423 {
9424         int i;
9425
9426         clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state);
9427         for (i = 0; i < bp->cp_nr_rings; i++) {
9428                 struct bnxt_napi *bnapi = bp->bnapi[i];
9429                 struct bnxt_cp_ring_info *cpr;
9430
9431                 cpr = &bnapi->cp_ring;
9432                 if (bnapi->in_reset)
9433                         cpr->sw_stats.rx.rx_resets++;
9434                 bnapi->in_reset = false;
9435
9436                 if (bnapi->rx_ring) {
9437                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
9438                         cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
9439                 }
9440                 napi_enable(&bnapi->napi);
9441         }
9442 }
9443
9444 void bnxt_tx_disable(struct bnxt *bp)
9445 {
9446         int i;
9447         struct bnxt_tx_ring_info *txr;
9448
9449         if (bp->tx_ring) {
9450                 for (i = 0; i < bp->tx_nr_rings; i++) {
9451                         txr = &bp->tx_ring[i];
9452                         WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING);
9453                 }
9454         }
9455         /* Make sure napi polls see @dev_state change */
9456         synchronize_net();
9457         /* Drop carrier first to prevent TX timeout */
9458         netif_carrier_off(bp->dev);
9459         /* Stop all TX queues */
9460         netif_tx_disable(bp->dev);
9461 }
9462
9463 void bnxt_tx_enable(struct bnxt *bp)
9464 {
9465         int i;
9466         struct bnxt_tx_ring_info *txr;
9467
9468         for (i = 0; i < bp->tx_nr_rings; i++) {
9469                 txr = &bp->tx_ring[i];
9470                 WRITE_ONCE(txr->dev_state, 0);
9471         }
9472         /* Make sure napi polls see @dev_state change */
9473         synchronize_net();
9474         netif_tx_wake_all_queues(bp->dev);
9475         if (BNXT_LINK_IS_UP(bp))
9476                 netif_carrier_on(bp->dev);
9477 }
9478
9479 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
9480 {
9481         u8 active_fec = link_info->active_fec_sig_mode &
9482                         PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
9483
9484         switch (active_fec) {
9485         default:
9486         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
9487                 return "None";
9488         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
9489                 return "Clause 74 BaseR";
9490         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
9491                 return "Clause 91 RS(528,514)";
9492         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
9493                 return "Clause 91 RS544_1XN";
9494         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
9495                 return "Clause 91 RS(544,514)";
9496         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
9497                 return "Clause 91 RS272_1XN";
9498         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
9499                 return "Clause 91 RS(272,257)";
9500         }
9501 }
9502
9503 void bnxt_report_link(struct bnxt *bp)
9504 {
9505         if (BNXT_LINK_IS_UP(bp)) {
9506                 const char *signal = "";
9507                 const char *flow_ctrl;
9508                 const char *duplex;
9509                 u32 speed;
9510                 u16 fec;
9511
9512                 netif_carrier_on(bp->dev);
9513                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
9514                 if (speed == SPEED_UNKNOWN) {
9515                         netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
9516                         return;
9517                 }
9518                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
9519                         duplex = "full";
9520                 else
9521                         duplex = "half";
9522                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
9523                         flow_ctrl = "ON - receive & transmit";
9524                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
9525                         flow_ctrl = "ON - transmit";
9526                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
9527                         flow_ctrl = "ON - receive";
9528                 else
9529                         flow_ctrl = "none";
9530                 if (bp->link_info.phy_qcfg_resp.option_flags &
9531                     PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) {
9532                         u8 sig_mode = bp->link_info.active_fec_sig_mode &
9533                                       PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK;
9534                         switch (sig_mode) {
9535                         case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ:
9536                                 signal = "(NRZ) ";
9537                                 break;
9538                         case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4:
9539                                 signal = "(PAM4) ";
9540                                 break;
9541                         default:
9542                                 break;
9543                         }
9544                 }
9545                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n",
9546                             speed, signal, duplex, flow_ctrl);
9547                 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP)
9548                         netdev_info(bp->dev, "EEE is %s\n",
9549                                     bp->eee.eee_active ? "active" :
9550                                                          "not active");
9551                 fec = bp->link_info.fec_cfg;
9552                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
9553                         netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
9554                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
9555                                     bnxt_report_fec(&bp->link_info));
9556         } else {
9557                 netif_carrier_off(bp->dev);
9558                 netdev_err(bp->dev, "NIC Link is Down\n");
9559         }
9560 }
9561
9562 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
9563 {
9564         if (!resp->supported_speeds_auto_mode &&
9565             !resp->supported_speeds_force_mode &&
9566             !resp->supported_pam4_speeds_auto_mode &&
9567             !resp->supported_pam4_speeds_force_mode)
9568                 return true;
9569         return false;
9570 }
9571
9572 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
9573 {
9574         struct bnxt_link_info *link_info = &bp->link_info;
9575         struct hwrm_port_phy_qcaps_output *resp;
9576         struct hwrm_port_phy_qcaps_input *req;
9577         int rc = 0;
9578
9579         if (bp->hwrm_spec_code < 0x10201)
9580                 return 0;
9581
9582         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS);
9583         if (rc)
9584                 return rc;
9585
9586         resp = hwrm_req_hold(bp, req);
9587         rc = hwrm_req_send(bp, req);
9588         if (rc)
9589                 goto hwrm_phy_qcaps_exit;
9590
9591         bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8);
9592         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
9593                 struct ethtool_eee *eee = &bp->eee;
9594                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
9595
9596                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9597                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
9598                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
9599                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
9600                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
9601         }
9602
9603         if (bp->hwrm_spec_code >= 0x10a01) {
9604                 if (bnxt_phy_qcaps_no_speed(resp)) {
9605                         link_info->phy_state = BNXT_PHY_STATE_DISABLED;
9606                         netdev_warn(bp->dev, "Ethernet link disabled\n");
9607                 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
9608                         link_info->phy_state = BNXT_PHY_STATE_ENABLED;
9609                         netdev_info(bp->dev, "Ethernet link enabled\n");
9610                         /* Phy re-enabled, reprobe the speeds */
9611                         link_info->support_auto_speeds = 0;
9612                         link_info->support_pam4_auto_speeds = 0;
9613                 }
9614         }
9615         if (resp->supported_speeds_auto_mode)
9616                 link_info->support_auto_speeds =
9617                         le16_to_cpu(resp->supported_speeds_auto_mode);
9618         if (resp->supported_pam4_speeds_auto_mode)
9619                 link_info->support_pam4_auto_speeds =
9620                         le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
9621
9622         bp->port_count = resp->port_cnt;
9623
9624 hwrm_phy_qcaps_exit:
9625         hwrm_req_drop(bp, req);
9626         return rc;
9627 }
9628
9629 static bool bnxt_support_dropped(u16 advertising, u16 supported)
9630 {
9631         u16 diff = advertising ^ supported;
9632
9633         return ((supported | diff) != supported);
9634 }
9635
9636 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
9637 {
9638         struct bnxt_link_info *link_info = &bp->link_info;
9639         struct hwrm_port_phy_qcfg_output *resp;
9640         struct hwrm_port_phy_qcfg_input *req;
9641         u8 link_state = link_info->link_state;
9642         bool support_changed = false;
9643         int rc;
9644
9645         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG);
9646         if (rc)
9647                 return rc;
9648
9649         resp = hwrm_req_hold(bp, req);
9650         rc = hwrm_req_send(bp, req);
9651         if (rc) {
9652                 hwrm_req_drop(bp, req);
9653                 if (BNXT_VF(bp) && rc == -ENODEV) {
9654                         netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n");
9655                         rc = 0;
9656                 }
9657                 return rc;
9658         }
9659
9660         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
9661         link_info->phy_link_status = resp->link;
9662         link_info->duplex = resp->duplex_cfg;
9663         if (bp->hwrm_spec_code >= 0x10800)
9664                 link_info->duplex = resp->duplex_state;
9665         link_info->pause = resp->pause;
9666         link_info->auto_mode = resp->auto_mode;
9667         link_info->auto_pause_setting = resp->auto_pause;
9668         link_info->lp_pause = resp->link_partner_adv_pause;
9669         link_info->force_pause_setting = resp->force_pause;
9670         link_info->duplex_setting = resp->duplex_cfg;
9671         if (link_info->phy_link_status == BNXT_LINK_LINK)
9672                 link_info->link_speed = le16_to_cpu(resp->link_speed);
9673         else
9674                 link_info->link_speed = 0;
9675         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
9676         link_info->force_pam4_link_speed =
9677                 le16_to_cpu(resp->force_pam4_link_speed);
9678         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
9679         link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
9680         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
9681         link_info->auto_pam4_link_speeds =
9682                 le16_to_cpu(resp->auto_pam4_link_speed_mask);
9683         link_info->lp_auto_link_speeds =
9684                 le16_to_cpu(resp->link_partner_adv_speeds);
9685         link_info->lp_auto_pam4_link_speeds =
9686                 resp->link_partner_pam4_adv_speeds;
9687         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
9688         link_info->phy_ver[0] = resp->phy_maj;
9689         link_info->phy_ver[1] = resp->phy_min;
9690         link_info->phy_ver[2] = resp->phy_bld;
9691         link_info->media_type = resp->media_type;
9692         link_info->phy_type = resp->phy_type;
9693         link_info->transceiver = resp->xcvr_pkg_type;
9694         link_info->phy_addr = resp->eee_config_phy_addr &
9695                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
9696         link_info->module_status = resp->module_status;
9697
9698         if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) {
9699                 struct ethtool_eee *eee = &bp->eee;
9700                 u16 fw_speeds;
9701
9702                 eee->eee_active = 0;
9703                 if (resp->eee_config_phy_addr &
9704                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
9705                         eee->eee_active = 1;
9706                         fw_speeds = le16_to_cpu(
9707                                 resp->link_partner_adv_eee_link_speed_mask);
9708                         eee->lp_advertised =
9709                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9710                 }
9711
9712                 /* Pull initial EEE config */
9713                 if (!chng_link_state) {
9714                         if (resp->eee_config_phy_addr &
9715                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
9716                                 eee->eee_enabled = 1;
9717
9718                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
9719                         eee->advertised =
9720                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9721
9722                         if (resp->eee_config_phy_addr &
9723                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
9724                                 __le32 tmr;
9725
9726                                 eee->tx_lpi_enabled = 1;
9727                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
9728                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
9729                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
9730                         }
9731                 }
9732         }
9733
9734         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
9735         if (bp->hwrm_spec_code >= 0x10504) {
9736                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
9737                 link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
9738         }
9739         /* TODO: need to add more logic to report VF link */
9740         if (chng_link_state) {
9741                 if (link_info->phy_link_status == BNXT_LINK_LINK)
9742                         link_info->link_state = BNXT_LINK_STATE_UP;
9743                 else
9744                         link_info->link_state = BNXT_LINK_STATE_DOWN;
9745                 if (link_state != link_info->link_state)
9746                         bnxt_report_link(bp);
9747         } else {
9748                 /* always link down if not require to update link state */
9749                 link_info->link_state = BNXT_LINK_STATE_DOWN;
9750         }
9751         hwrm_req_drop(bp, req);
9752
9753         if (!BNXT_PHY_CFG_ABLE(bp))
9754                 return 0;
9755
9756         /* Check if any advertised speeds are no longer supported. The caller
9757          * holds the link_lock mutex, so we can modify link_info settings.
9758          */
9759         if (bnxt_support_dropped(link_info->advertising,
9760                                  link_info->support_auto_speeds)) {
9761                 link_info->advertising = link_info->support_auto_speeds;
9762                 support_changed = true;
9763         }
9764         if (bnxt_support_dropped(link_info->advertising_pam4,
9765                                  link_info->support_pam4_auto_speeds)) {
9766                 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
9767                 support_changed = true;
9768         }
9769         if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
9770                 bnxt_hwrm_set_link_setting(bp, true, false);
9771         return 0;
9772 }
9773
9774 static void bnxt_get_port_module_status(struct bnxt *bp)
9775 {
9776         struct bnxt_link_info *link_info = &bp->link_info;
9777         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
9778         u8 module_status;
9779
9780         if (bnxt_update_link(bp, true))
9781                 return;
9782
9783         module_status = link_info->module_status;
9784         switch (module_status) {
9785         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
9786         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
9787         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
9788                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
9789                             bp->pf.port_id);
9790                 if (bp->hwrm_spec_code >= 0x10201) {
9791                         netdev_warn(bp->dev, "Module part number %s\n",
9792                                     resp->phy_vendor_partnumber);
9793                 }
9794                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
9795                         netdev_warn(bp->dev, "TX is disabled\n");
9796                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
9797                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
9798         }
9799 }
9800
9801 static void
9802 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9803 {
9804         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
9805                 if (bp->hwrm_spec_code >= 0x10201)
9806                         req->auto_pause =
9807                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
9808                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9809                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
9810                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9811                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
9812                 req->enables |=
9813                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9814         } else {
9815                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9816                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
9817                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9818                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9819                 req->enables |=
9820                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9821                 if (bp->hwrm_spec_code >= 0x10201) {
9822                         req->auto_pause = req->force_pause;
9823                         req->enables |= cpu_to_le32(
9824                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9825                 }
9826         }
9827 }
9828
9829 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9830 {
9831         if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
9832                 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9833                 if (bp->link_info.advertising) {
9834                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9835                         req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
9836                 }
9837                 if (bp->link_info.advertising_pam4) {
9838                         req->enables |=
9839                                 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
9840                         req->auto_link_pam4_speed_mask =
9841                                 cpu_to_le16(bp->link_info.advertising_pam4);
9842                 }
9843                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9844                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9845         } else {
9846                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9847                 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
9848                         req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9849                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
9850                 } else {
9851                         req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9852                 }
9853         }
9854
9855         /* tell chimp that the setting takes effect immediately */
9856         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9857 }
9858
9859 int bnxt_hwrm_set_pause(struct bnxt *bp)
9860 {
9861         struct hwrm_port_phy_cfg_input *req;
9862         int rc;
9863
9864         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9865         if (rc)
9866                 return rc;
9867
9868         bnxt_hwrm_set_pause_common(bp, req);
9869
9870         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9871             bp->link_info.force_link_chng)
9872                 bnxt_hwrm_set_link_common(bp, req);
9873
9874         rc = hwrm_req_send(bp, req);
9875         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9876                 /* since changing of pause setting doesn't trigger any link
9877                  * change event, the driver needs to update the current pause
9878                  * result upon successfully return of the phy_cfg command
9879                  */
9880                 bp->link_info.pause =
9881                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9882                 bp->link_info.auto_pause_setting = 0;
9883                 if (!bp->link_info.force_link_chng)
9884                         bnxt_report_link(bp);
9885         }
9886         bp->link_info.force_link_chng = false;
9887         return rc;
9888 }
9889
9890 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9891                               struct hwrm_port_phy_cfg_input *req)
9892 {
9893         struct ethtool_eee *eee = &bp->eee;
9894
9895         if (eee->eee_enabled) {
9896                 u16 eee_speeds;
9897                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9898
9899                 if (eee->tx_lpi_enabled)
9900                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9901                 else
9902                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9903
9904                 req->flags |= cpu_to_le32(flags);
9905                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9906                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9907                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9908         } else {
9909                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9910         }
9911 }
9912
9913 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9914 {
9915         struct hwrm_port_phy_cfg_input *req;
9916         int rc;
9917
9918         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9919         if (rc)
9920                 return rc;
9921
9922         if (set_pause)
9923                 bnxt_hwrm_set_pause_common(bp, req);
9924
9925         bnxt_hwrm_set_link_common(bp, req);
9926
9927         if (set_eee)
9928                 bnxt_hwrm_set_eee(bp, req);
9929         return hwrm_req_send(bp, req);
9930 }
9931
9932 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9933 {
9934         struct hwrm_port_phy_cfg_input *req;
9935         int rc;
9936
9937         if (!BNXT_SINGLE_PF(bp))
9938                 return 0;
9939
9940         if (pci_num_vf(bp->pdev) &&
9941             !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN))
9942                 return 0;
9943
9944         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9945         if (rc)
9946                 return rc;
9947
9948         req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9949         rc = hwrm_req_send(bp, req);
9950         if (!rc) {
9951                 mutex_lock(&bp->link_lock);
9952                 /* Device is not obliged link down in certain scenarios, even
9953                  * when forced. Setting the state unknown is consistent with
9954                  * driver startup and will force link state to be reported
9955                  * during subsequent open based on PORT_PHY_QCFG.
9956                  */
9957                 bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN;
9958                 mutex_unlock(&bp->link_lock);
9959         }
9960         return rc;
9961 }
9962
9963 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
9964 {
9965 #ifdef CONFIG_TEE_BNXT_FW
9966         int rc = tee_bnxt_fw_load();
9967
9968         if (rc)
9969                 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
9970
9971         return rc;
9972 #else
9973         netdev_err(bp->dev, "OP-TEE not supported\n");
9974         return -ENODEV;
9975 #endif
9976 }
9977
9978 static int bnxt_try_recover_fw(struct bnxt *bp)
9979 {
9980         if (bp->fw_health && bp->fw_health->status_reliable) {
9981                 int retry = 0, rc;
9982                 u32 sts;
9983
9984                 do {
9985                         sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
9986                         rc = bnxt_hwrm_poll(bp);
9987                         if (!BNXT_FW_IS_BOOTING(sts) &&
9988                             !BNXT_FW_IS_RECOVERING(sts))
9989                                 break;
9990                         retry++;
9991                 } while (rc == -EBUSY && retry < BNXT_FW_RETRY);
9992
9993                 if (!BNXT_FW_IS_HEALTHY(sts)) {
9994                         netdev_err(bp->dev,
9995                                    "Firmware not responding, status: 0x%x\n",
9996                                    sts);
9997                         rc = -ENODEV;
9998                 }
9999                 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
10000                         netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
10001                         return bnxt_fw_reset_via_optee(bp);
10002                 }
10003                 return rc;
10004         }
10005
10006         return -ENODEV;
10007 }
10008
10009 static void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset)
10010 {
10011         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10012
10013         if (!BNXT_NEW_RM(bp))
10014                 return; /* no resource reservations required */
10015
10016         hw_resc->resv_cp_rings = 0;
10017         hw_resc->resv_stat_ctxs = 0;
10018         hw_resc->resv_irqs = 0;
10019         hw_resc->resv_tx_rings = 0;
10020         hw_resc->resv_rx_rings = 0;
10021         hw_resc->resv_hw_ring_grps = 0;
10022         hw_resc->resv_vnics = 0;
10023         if (!fw_reset) {
10024                 bp->tx_nr_rings = 0;
10025                 bp->rx_nr_rings = 0;
10026         }
10027 }
10028
10029 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset)
10030 {
10031         int rc;
10032
10033         if (!BNXT_NEW_RM(bp))
10034                 return 0; /* no resource reservations required */
10035
10036         rc = bnxt_hwrm_func_resc_qcaps(bp, true);
10037         if (rc)
10038                 netdev_err(bp->dev, "resc_qcaps failed\n");
10039
10040         bnxt_clear_reservations(bp, fw_reset);
10041
10042         return rc;
10043 }
10044
10045 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
10046 {
10047         struct hwrm_func_drv_if_change_output *resp;
10048         struct hwrm_func_drv_if_change_input *req;
10049         bool fw_reset = !bp->irq_tbl;
10050         bool resc_reinit = false;
10051         int rc, retry = 0;
10052         u32 flags = 0;
10053
10054         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
10055                 return 0;
10056
10057         rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE);
10058         if (rc)
10059                 return rc;
10060
10061         if (up)
10062                 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
10063         resp = hwrm_req_hold(bp, req);
10064
10065         hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
10066         while (retry < BNXT_FW_IF_RETRY) {
10067                 rc = hwrm_req_send(bp, req);
10068                 if (rc != -EAGAIN)
10069                         break;
10070
10071                 msleep(50);
10072                 retry++;
10073         }
10074
10075         if (rc == -EAGAIN) {
10076                 hwrm_req_drop(bp, req);
10077                 return rc;
10078         } else if (!rc) {
10079                 flags = le32_to_cpu(resp->flags);
10080         } else if (up) {
10081                 rc = bnxt_try_recover_fw(bp);
10082                 fw_reset = true;
10083         }
10084         hwrm_req_drop(bp, req);
10085         if (rc)
10086                 return rc;
10087
10088         if (!up) {
10089                 bnxt_inv_fw_health_reg(bp);
10090                 return 0;
10091         }
10092
10093         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
10094                 resc_reinit = true;
10095         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE ||
10096             test_bit(BNXT_STATE_FW_RESET_DET, &bp->state))
10097                 fw_reset = true;
10098         else
10099                 bnxt_remap_fw_health_regs(bp);
10100
10101         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
10102                 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
10103                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10104                 return -ENODEV;
10105         }
10106         if (resc_reinit || fw_reset) {
10107                 if (fw_reset) {
10108                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10109                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10110                                 bnxt_ulp_stop(bp);
10111                         bnxt_free_ctx_mem(bp);
10112                         kfree(bp->ctx);
10113                         bp->ctx = NULL;
10114                         bnxt_dcb_free(bp);
10115                         rc = bnxt_fw_init_one(bp);
10116                         if (rc) {
10117                                 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10118                                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10119                                 return rc;
10120                         }
10121                         bnxt_clear_int_mode(bp);
10122                         rc = bnxt_init_int_mode(bp);
10123                         if (rc) {
10124                                 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10125                                 netdev_err(bp->dev, "init int mode failed\n");
10126                                 return rc;
10127                         }
10128                 }
10129                 rc = bnxt_cancel_reservations(bp, fw_reset);
10130         }
10131         return rc;
10132 }
10133
10134 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
10135 {
10136         struct hwrm_port_led_qcaps_output *resp;
10137         struct hwrm_port_led_qcaps_input *req;
10138         struct bnxt_pf_info *pf = &bp->pf;
10139         int rc;
10140
10141         bp->num_leds = 0;
10142         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
10143                 return 0;
10144
10145         rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS);
10146         if (rc)
10147                 return rc;
10148
10149         req->port_id = cpu_to_le16(pf->port_id);
10150         resp = hwrm_req_hold(bp, req);
10151         rc = hwrm_req_send(bp, req);
10152         if (rc) {
10153                 hwrm_req_drop(bp, req);
10154                 return rc;
10155         }
10156         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
10157                 int i;
10158
10159                 bp->num_leds = resp->num_leds;
10160                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
10161                                                  bp->num_leds);
10162                 for (i = 0; i < bp->num_leds; i++) {
10163                         struct bnxt_led_info *led = &bp->leds[i];
10164                         __le16 caps = led->led_state_caps;
10165
10166                         if (!led->led_group_id ||
10167                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
10168                                 bp->num_leds = 0;
10169                                 break;
10170                         }
10171                 }
10172         }
10173         hwrm_req_drop(bp, req);
10174         return 0;
10175 }
10176
10177 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
10178 {
10179         struct hwrm_wol_filter_alloc_output *resp;
10180         struct hwrm_wol_filter_alloc_input *req;
10181         int rc;
10182
10183         rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC);
10184         if (rc)
10185                 return rc;
10186
10187         req->port_id = cpu_to_le16(bp->pf.port_id);
10188         req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
10189         req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
10190         memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN);
10191
10192         resp = hwrm_req_hold(bp, req);
10193         rc = hwrm_req_send(bp, req);
10194         if (!rc)
10195                 bp->wol_filter_id = resp->wol_filter_id;
10196         hwrm_req_drop(bp, req);
10197         return rc;
10198 }
10199
10200 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
10201 {
10202         struct hwrm_wol_filter_free_input *req;
10203         int rc;
10204
10205         rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE);
10206         if (rc)
10207                 return rc;
10208
10209         req->port_id = cpu_to_le16(bp->pf.port_id);
10210         req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
10211         req->wol_filter_id = bp->wol_filter_id;
10212
10213         return hwrm_req_send(bp, req);
10214 }
10215
10216 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
10217 {
10218         struct hwrm_wol_filter_qcfg_output *resp;
10219         struct hwrm_wol_filter_qcfg_input *req;
10220         u16 next_handle = 0;
10221         int rc;
10222
10223         rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG);
10224         if (rc)
10225                 return rc;
10226
10227         req->port_id = cpu_to_le16(bp->pf.port_id);
10228         req->handle = cpu_to_le16(handle);
10229         resp = hwrm_req_hold(bp, req);
10230         rc = hwrm_req_send(bp, req);
10231         if (!rc) {
10232                 next_handle = le16_to_cpu(resp->next_handle);
10233                 if (next_handle != 0) {
10234                         if (resp->wol_type ==
10235                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
10236                                 bp->wol = 1;
10237                                 bp->wol_filter_id = resp->wol_filter_id;
10238                         }
10239                 }
10240         }
10241         hwrm_req_drop(bp, req);
10242         return next_handle;
10243 }
10244
10245 static void bnxt_get_wol_settings(struct bnxt *bp)
10246 {
10247         u16 handle = 0;
10248
10249         bp->wol = 0;
10250         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
10251                 return;
10252
10253         do {
10254                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
10255         } while (handle && handle != 0xffff);
10256 }
10257
10258 #ifdef CONFIG_BNXT_HWMON
10259 static ssize_t bnxt_show_temp(struct device *dev,
10260                               struct device_attribute *devattr, char *buf)
10261 {
10262         struct hwrm_temp_monitor_query_output *resp;
10263         struct hwrm_temp_monitor_query_input *req;
10264         struct bnxt *bp = dev_get_drvdata(dev);
10265         u32 len = 0;
10266         int rc;
10267
10268         rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY);
10269         if (rc)
10270                 return rc;
10271         resp = hwrm_req_hold(bp, req);
10272         rc = hwrm_req_send(bp, req);
10273         if (!rc)
10274                 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
10275         hwrm_req_drop(bp, req);
10276         if (rc)
10277                 return rc;
10278         return len;
10279 }
10280 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
10281
10282 static struct attribute *bnxt_attrs[] = {
10283         &sensor_dev_attr_temp1_input.dev_attr.attr,
10284         NULL
10285 };
10286 ATTRIBUTE_GROUPS(bnxt);
10287
10288 static void bnxt_hwmon_close(struct bnxt *bp)
10289 {
10290         if (bp->hwmon_dev) {
10291                 hwmon_device_unregister(bp->hwmon_dev);
10292                 bp->hwmon_dev = NULL;
10293         }
10294 }
10295
10296 static void bnxt_hwmon_open(struct bnxt *bp)
10297 {
10298         struct hwrm_temp_monitor_query_input *req;
10299         struct pci_dev *pdev = bp->pdev;
10300         int rc;
10301
10302         rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY);
10303         if (!rc)
10304                 rc = hwrm_req_send_silent(bp, req);
10305         if (rc == -EACCES || rc == -EOPNOTSUPP) {
10306                 bnxt_hwmon_close(bp);
10307                 return;
10308         }
10309
10310         if (bp->hwmon_dev)
10311                 return;
10312
10313         bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
10314                                                           DRV_MODULE_NAME, bp,
10315                                                           bnxt_groups);
10316         if (IS_ERR(bp->hwmon_dev)) {
10317                 bp->hwmon_dev = NULL;
10318                 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
10319         }
10320 }
10321 #else
10322 static void bnxt_hwmon_close(struct bnxt *bp)
10323 {
10324 }
10325
10326 static void bnxt_hwmon_open(struct bnxt *bp)
10327 {
10328 }
10329 #endif
10330
10331 static bool bnxt_eee_config_ok(struct bnxt *bp)
10332 {
10333         struct ethtool_eee *eee = &bp->eee;
10334         struct bnxt_link_info *link_info = &bp->link_info;
10335
10336         if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP))
10337                 return true;
10338
10339         if (eee->eee_enabled) {
10340                 u32 advertising =
10341                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
10342
10343                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
10344                         eee->eee_enabled = 0;
10345                         return false;
10346                 }
10347                 if (eee->advertised & ~advertising) {
10348                         eee->advertised = advertising & eee->supported;
10349                         return false;
10350                 }
10351         }
10352         return true;
10353 }
10354
10355 static int bnxt_update_phy_setting(struct bnxt *bp)
10356 {
10357         int rc;
10358         bool update_link = false;
10359         bool update_pause = false;
10360         bool update_eee = false;
10361         struct bnxt_link_info *link_info = &bp->link_info;
10362
10363         rc = bnxt_update_link(bp, true);
10364         if (rc) {
10365                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
10366                            rc);
10367                 return rc;
10368         }
10369         if (!BNXT_SINGLE_PF(bp))
10370                 return 0;
10371
10372         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
10373             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
10374             link_info->req_flow_ctrl)
10375                 update_pause = true;
10376         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
10377             link_info->force_pause_setting != link_info->req_flow_ctrl)
10378                 update_pause = true;
10379         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
10380                 if (BNXT_AUTO_MODE(link_info->auto_mode))
10381                         update_link = true;
10382                 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
10383                     link_info->req_link_speed != link_info->force_link_speed)
10384                         update_link = true;
10385                 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
10386                          link_info->req_link_speed != link_info->force_pam4_link_speed)
10387                         update_link = true;
10388                 if (link_info->req_duplex != link_info->duplex_setting)
10389                         update_link = true;
10390         } else {
10391                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
10392                         update_link = true;
10393                 if (link_info->advertising != link_info->auto_link_speeds ||
10394                     link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
10395                         update_link = true;
10396         }
10397
10398         /* The last close may have shutdown the link, so need to call
10399          * PHY_CFG to bring it back up.
10400          */
10401         if (!BNXT_LINK_IS_UP(bp))
10402                 update_link = true;
10403
10404         if (!bnxt_eee_config_ok(bp))
10405                 update_eee = true;
10406
10407         if (update_link)
10408                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
10409         else if (update_pause)
10410                 rc = bnxt_hwrm_set_pause(bp);
10411         if (rc) {
10412                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
10413                            rc);
10414                 return rc;
10415         }
10416
10417         return rc;
10418 }
10419
10420 /* Common routine to pre-map certain register block to different GRC window.
10421  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
10422  * in PF and 3 windows in VF that can be customized to map in different
10423  * register blocks.
10424  */
10425 static void bnxt_preset_reg_win(struct bnxt *bp)
10426 {
10427         if (BNXT_PF(bp)) {
10428                 /* CAG registers map to GRC window #4 */
10429                 writel(BNXT_CAG_REG_BASE,
10430                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
10431         }
10432 }
10433
10434 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
10435
10436 static int bnxt_reinit_after_abort(struct bnxt *bp)
10437 {
10438         int rc;
10439
10440         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10441                 return -EBUSY;
10442
10443         if (bp->dev->reg_state == NETREG_UNREGISTERED)
10444                 return -ENODEV;
10445
10446         rc = bnxt_fw_init_one(bp);
10447         if (!rc) {
10448                 bnxt_clear_int_mode(bp);
10449                 rc = bnxt_init_int_mode(bp);
10450                 if (!rc) {
10451                         clear_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10452                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10453                 }
10454         }
10455         return rc;
10456 }
10457
10458 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10459 {
10460         int rc = 0;
10461
10462         bnxt_preset_reg_win(bp);
10463         netif_carrier_off(bp->dev);
10464         if (irq_re_init) {
10465                 /* Reserve rings now if none were reserved at driver probe. */
10466                 rc = bnxt_init_dflt_ring_mode(bp);
10467                 if (rc) {
10468                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
10469                         return rc;
10470                 }
10471         }
10472         rc = bnxt_reserve_rings(bp, irq_re_init);
10473         if (rc)
10474                 return rc;
10475         if ((bp->flags & BNXT_FLAG_RFS) &&
10476             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
10477                 /* disable RFS if falling back to INTA */
10478                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
10479                 bp->flags &= ~BNXT_FLAG_RFS;
10480         }
10481
10482         rc = bnxt_alloc_mem(bp, irq_re_init);
10483         if (rc) {
10484                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
10485                 goto open_err_free_mem;
10486         }
10487
10488         if (irq_re_init) {
10489                 bnxt_init_napi(bp);
10490                 rc = bnxt_request_irq(bp);
10491                 if (rc) {
10492                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
10493                         goto open_err_irq;
10494                 }
10495         }
10496
10497         rc = bnxt_init_nic(bp, irq_re_init);
10498         if (rc) {
10499                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
10500                 goto open_err_irq;
10501         }
10502
10503         bnxt_enable_napi(bp);
10504         bnxt_debug_dev_init(bp);
10505
10506         if (link_re_init) {
10507                 mutex_lock(&bp->link_lock);
10508                 rc = bnxt_update_phy_setting(bp);
10509                 mutex_unlock(&bp->link_lock);
10510                 if (rc) {
10511                         netdev_warn(bp->dev, "failed to update phy settings\n");
10512                         if (BNXT_SINGLE_PF(bp)) {
10513                                 bp->link_info.phy_retry = true;
10514                                 bp->link_info.phy_retry_expires =
10515                                         jiffies + 5 * HZ;
10516                         }
10517                 }
10518         }
10519
10520         if (irq_re_init)
10521                 udp_tunnel_nic_reset_ntf(bp->dev);
10522
10523         if (bp->tx_nr_rings_xdp < num_possible_cpus()) {
10524                 if (!static_key_enabled(&bnxt_xdp_locking_key))
10525                         static_branch_enable(&bnxt_xdp_locking_key);
10526         } else if (static_key_enabled(&bnxt_xdp_locking_key)) {
10527                 static_branch_disable(&bnxt_xdp_locking_key);
10528         }
10529         set_bit(BNXT_STATE_OPEN, &bp->state);
10530         bnxt_enable_int(bp);
10531         /* Enable TX queues */
10532         bnxt_tx_enable(bp);
10533         mod_timer(&bp->timer, jiffies + bp->current_interval);
10534         /* Poll link status and check for SFP+ module status */
10535         mutex_lock(&bp->link_lock);
10536         bnxt_get_port_module_status(bp);
10537         mutex_unlock(&bp->link_lock);
10538
10539         /* VF-reps may need to be re-opened after the PF is re-opened */
10540         if (BNXT_PF(bp))
10541                 bnxt_vf_reps_open(bp);
10542         bnxt_ptp_init_rtc(bp, true);
10543         bnxt_ptp_cfg_tstamp_filters(bp);
10544         return 0;
10545
10546 open_err_irq:
10547         bnxt_del_napi(bp);
10548
10549 open_err_free_mem:
10550         bnxt_free_skbs(bp);
10551         bnxt_free_irq(bp);
10552         bnxt_free_mem(bp, true);
10553         return rc;
10554 }
10555
10556 /* rtnl_lock held */
10557 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10558 {
10559         int rc = 0;
10560
10561         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
10562                 rc = -EIO;
10563         if (!rc)
10564                 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
10565         if (rc) {
10566                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
10567                 dev_close(bp->dev);
10568         }
10569         return rc;
10570 }
10571
10572 /* rtnl_lock held, open the NIC half way by allocating all resources, but
10573  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
10574  * self tests.
10575  */
10576 int bnxt_half_open_nic(struct bnxt *bp)
10577 {
10578         int rc = 0;
10579
10580         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
10581                 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n");
10582                 rc = -ENODEV;
10583                 goto half_open_err;
10584         }
10585
10586         rc = bnxt_alloc_mem(bp, true);
10587         if (rc) {
10588                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
10589                 goto half_open_err;
10590         }
10591         set_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10592         rc = bnxt_init_nic(bp, true);
10593         if (rc) {
10594                 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10595                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
10596                 goto half_open_err;
10597         }
10598         return 0;
10599
10600 half_open_err:
10601         bnxt_free_skbs(bp);
10602         bnxt_free_mem(bp, true);
10603         dev_close(bp->dev);
10604         return rc;
10605 }
10606
10607 /* rtnl_lock held, this call can only be made after a previous successful
10608  * call to bnxt_half_open_nic().
10609  */
10610 void bnxt_half_close_nic(struct bnxt *bp)
10611 {
10612         bnxt_hwrm_resource_free(bp, false, true);
10613         bnxt_free_skbs(bp);
10614         bnxt_free_mem(bp, true);
10615         clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10616 }
10617
10618 void bnxt_reenable_sriov(struct bnxt *bp)
10619 {
10620         if (BNXT_PF(bp)) {
10621                 struct bnxt_pf_info *pf = &bp->pf;
10622                 int n = pf->active_vfs;
10623
10624                 if (n)
10625                         bnxt_cfg_hw_sriov(bp, &n, true);
10626         }
10627 }
10628
10629 static int bnxt_open(struct net_device *dev)
10630 {
10631         struct bnxt *bp = netdev_priv(dev);
10632         int rc;
10633
10634         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
10635                 rc = bnxt_reinit_after_abort(bp);
10636                 if (rc) {
10637                         if (rc == -EBUSY)
10638                                 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n");
10639                         else
10640                                 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n");
10641                         return -ENODEV;
10642                 }
10643         }
10644
10645         rc = bnxt_hwrm_if_change(bp, true);
10646         if (rc)
10647                 return rc;
10648
10649         rc = __bnxt_open_nic(bp, true, true);
10650         if (rc) {
10651                 bnxt_hwrm_if_change(bp, false);
10652         } else {
10653                 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
10654                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10655                                 bnxt_ulp_start(bp, 0);
10656                                 bnxt_reenable_sriov(bp);
10657                         }
10658                 }
10659                 bnxt_hwmon_open(bp);
10660         }
10661
10662         return rc;
10663 }
10664
10665 static bool bnxt_drv_busy(struct bnxt *bp)
10666 {
10667         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
10668                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
10669 }
10670
10671 static void bnxt_get_ring_stats(struct bnxt *bp,
10672                                 struct rtnl_link_stats64 *stats);
10673
10674 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
10675                              bool link_re_init)
10676 {
10677         /* Close the VF-reps before closing PF */
10678         if (BNXT_PF(bp))
10679                 bnxt_vf_reps_close(bp);
10680
10681         /* Change device state to avoid TX queue wake up's */
10682         bnxt_tx_disable(bp);
10683
10684         clear_bit(BNXT_STATE_OPEN, &bp->state);
10685         smp_mb__after_atomic();
10686         while (bnxt_drv_busy(bp))
10687                 msleep(20);
10688
10689         /* Flush rings and disable interrupts */
10690         bnxt_shutdown_nic(bp, irq_re_init);
10691
10692         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
10693
10694         bnxt_debug_dev_exit(bp);
10695         bnxt_disable_napi(bp);
10696         del_timer_sync(&bp->timer);
10697         bnxt_free_skbs(bp);
10698
10699         /* Save ring stats before shutdown */
10700         if (bp->bnapi && irq_re_init)
10701                 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
10702         if (irq_re_init) {
10703                 bnxt_free_irq(bp);
10704                 bnxt_del_napi(bp);
10705         }
10706         bnxt_free_mem(bp, irq_re_init);
10707 }
10708
10709 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10710 {
10711         int rc = 0;
10712
10713         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10714                 /* If we get here, it means firmware reset is in progress
10715                  * while we are trying to close.  We can safely proceed with
10716                  * the close because we are holding rtnl_lock().  Some firmware
10717                  * messages may fail as we proceed to close.  We set the
10718                  * ABORT_ERR flag here so that the FW reset thread will later
10719                  * abort when it gets the rtnl_lock() and sees the flag.
10720                  */
10721                 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
10722                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10723         }
10724
10725 #ifdef CONFIG_BNXT_SRIOV
10726         if (bp->sriov_cfg) {
10727                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
10728                                                       !bp->sriov_cfg,
10729                                                       BNXT_SRIOV_CFG_WAIT_TMO);
10730                 if (rc)
10731                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
10732         }
10733 #endif
10734         __bnxt_close_nic(bp, irq_re_init, link_re_init);
10735         return rc;
10736 }
10737
10738 static int bnxt_close(struct net_device *dev)
10739 {
10740         struct bnxt *bp = netdev_priv(dev);
10741
10742         bnxt_hwmon_close(bp);
10743         bnxt_close_nic(bp, true, true);
10744         bnxt_hwrm_shutdown_link(bp);
10745         bnxt_hwrm_if_change(bp, false);
10746         return 0;
10747 }
10748
10749 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
10750                                    u16 *val)
10751 {
10752         struct hwrm_port_phy_mdio_read_output *resp;
10753         struct hwrm_port_phy_mdio_read_input *req;
10754         int rc;
10755
10756         if (bp->hwrm_spec_code < 0x10a00)
10757                 return -EOPNOTSUPP;
10758
10759         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ);
10760         if (rc)
10761                 return rc;
10762
10763         req->port_id = cpu_to_le16(bp->pf.port_id);
10764         req->phy_addr = phy_addr;
10765         req->reg_addr = cpu_to_le16(reg & 0x1f);
10766         if (mdio_phy_id_is_c45(phy_addr)) {
10767                 req->cl45_mdio = 1;
10768                 req->phy_addr = mdio_phy_id_prtad(phy_addr);
10769                 req->dev_addr = mdio_phy_id_devad(phy_addr);
10770                 req->reg_addr = cpu_to_le16(reg);
10771         }
10772
10773         resp = hwrm_req_hold(bp, req);
10774         rc = hwrm_req_send(bp, req);
10775         if (!rc)
10776                 *val = le16_to_cpu(resp->reg_data);
10777         hwrm_req_drop(bp, req);
10778         return rc;
10779 }
10780
10781 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
10782                                     u16 val)
10783 {
10784         struct hwrm_port_phy_mdio_write_input *req;
10785         int rc;
10786
10787         if (bp->hwrm_spec_code < 0x10a00)
10788                 return -EOPNOTSUPP;
10789
10790         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE);
10791         if (rc)
10792                 return rc;
10793
10794         req->port_id = cpu_to_le16(bp->pf.port_id);
10795         req->phy_addr = phy_addr;
10796         req->reg_addr = cpu_to_le16(reg & 0x1f);
10797         if (mdio_phy_id_is_c45(phy_addr)) {
10798                 req->cl45_mdio = 1;
10799                 req->phy_addr = mdio_phy_id_prtad(phy_addr);
10800                 req->dev_addr = mdio_phy_id_devad(phy_addr);
10801                 req->reg_addr = cpu_to_le16(reg);
10802         }
10803         req->reg_data = cpu_to_le16(val);
10804
10805         return hwrm_req_send(bp, req);
10806 }
10807
10808 /* rtnl_lock held */
10809 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
10810 {
10811         struct mii_ioctl_data *mdio = if_mii(ifr);
10812         struct bnxt *bp = netdev_priv(dev);
10813         int rc;
10814
10815         switch (cmd) {
10816         case SIOCGMIIPHY:
10817                 mdio->phy_id = bp->link_info.phy_addr;
10818
10819                 fallthrough;
10820         case SIOCGMIIREG: {
10821                 u16 mii_regval = 0;
10822
10823                 if (!netif_running(dev))
10824                         return -EAGAIN;
10825
10826                 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
10827                                              &mii_regval);
10828                 mdio->val_out = mii_regval;
10829                 return rc;
10830         }
10831
10832         case SIOCSMIIREG:
10833                 if (!netif_running(dev))
10834                         return -EAGAIN;
10835
10836                 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
10837                                                 mdio->val_in);
10838
10839         case SIOCSHWTSTAMP:
10840                 return bnxt_hwtstamp_set(dev, ifr);
10841
10842         case SIOCGHWTSTAMP:
10843                 return bnxt_hwtstamp_get(dev, ifr);
10844
10845         default:
10846                 /* do nothing */
10847                 break;
10848         }
10849         return -EOPNOTSUPP;
10850 }
10851
10852 static void bnxt_get_ring_stats(struct bnxt *bp,
10853                                 struct rtnl_link_stats64 *stats)
10854 {
10855         int i;
10856
10857         for (i = 0; i < bp->cp_nr_rings; i++) {
10858                 struct bnxt_napi *bnapi = bp->bnapi[i];
10859                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10860                 u64 *sw = cpr->stats.sw_stats;
10861
10862                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
10863                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10864                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
10865
10866                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
10867                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
10868                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
10869
10870                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
10871                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
10872                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
10873
10874                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
10875                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
10876                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
10877
10878                 stats->rx_missed_errors +=
10879                         BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
10880
10881                 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10882
10883                 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
10884
10885                 stats->rx_dropped +=
10886                         cpr->sw_stats.rx.rx_netpoll_discards +
10887                         cpr->sw_stats.rx.rx_oom_discards;
10888         }
10889 }
10890
10891 static void bnxt_add_prev_stats(struct bnxt *bp,
10892                                 struct rtnl_link_stats64 *stats)
10893 {
10894         struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
10895
10896         stats->rx_packets += prev_stats->rx_packets;
10897         stats->tx_packets += prev_stats->tx_packets;
10898         stats->rx_bytes += prev_stats->rx_bytes;
10899         stats->tx_bytes += prev_stats->tx_bytes;
10900         stats->rx_missed_errors += prev_stats->rx_missed_errors;
10901         stats->multicast += prev_stats->multicast;
10902         stats->rx_dropped += prev_stats->rx_dropped;
10903         stats->tx_dropped += prev_stats->tx_dropped;
10904 }
10905
10906 static void
10907 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
10908 {
10909         struct bnxt *bp = netdev_priv(dev);
10910
10911         set_bit(BNXT_STATE_READ_STATS, &bp->state);
10912         /* Make sure bnxt_close_nic() sees that we are reading stats before
10913          * we check the BNXT_STATE_OPEN flag.
10914          */
10915         smp_mb__after_atomic();
10916         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10917                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10918                 *stats = bp->net_stats_prev;
10919                 return;
10920         }
10921
10922         bnxt_get_ring_stats(bp, stats);
10923         bnxt_add_prev_stats(bp, stats);
10924
10925         if (bp->flags & BNXT_FLAG_PORT_STATS) {
10926                 u64 *rx = bp->port_stats.sw_stats;
10927                 u64 *tx = bp->port_stats.sw_stats +
10928                           BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10929
10930                 stats->rx_crc_errors =
10931                         BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
10932                 stats->rx_frame_errors =
10933                         BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
10934                 stats->rx_length_errors =
10935                         BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
10936                         BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
10937                         BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
10938                 stats->rx_errors =
10939                         BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
10940                         BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
10941                 stats->collisions =
10942                         BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
10943                 stats->tx_fifo_errors =
10944                         BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
10945                 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
10946         }
10947         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10948 }
10949
10950 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
10951 {
10952         struct net_device *dev = bp->dev;
10953         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10954         struct netdev_hw_addr *ha;
10955         u8 *haddr;
10956         int mc_count = 0;
10957         bool update = false;
10958         int off = 0;
10959
10960         netdev_for_each_mc_addr(ha, dev) {
10961                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
10962                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10963                         vnic->mc_list_count = 0;
10964                         return false;
10965                 }
10966                 haddr = ha->addr;
10967                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
10968                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
10969                         update = true;
10970                 }
10971                 off += ETH_ALEN;
10972                 mc_count++;
10973         }
10974         if (mc_count)
10975                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
10976
10977         if (mc_count != vnic->mc_list_count) {
10978                 vnic->mc_list_count = mc_count;
10979                 update = true;
10980         }
10981         return update;
10982 }
10983
10984 static bool bnxt_uc_list_updated(struct bnxt *bp)
10985 {
10986         struct net_device *dev = bp->dev;
10987         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10988         struct netdev_hw_addr *ha;
10989         int off = 0;
10990
10991         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
10992                 return true;
10993
10994         netdev_for_each_uc_addr(ha, dev) {
10995                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
10996                         return true;
10997
10998                 off += ETH_ALEN;
10999         }
11000         return false;
11001 }
11002
11003 static void bnxt_set_rx_mode(struct net_device *dev)
11004 {
11005         struct bnxt *bp = netdev_priv(dev);
11006         struct bnxt_vnic_info *vnic;
11007         bool mc_update = false;
11008         bool uc_update;
11009         u32 mask;
11010
11011         if (!test_bit(BNXT_STATE_OPEN, &bp->state))
11012                 return;
11013
11014         vnic = &bp->vnic_info[0];
11015         mask = vnic->rx_mask;
11016         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
11017                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
11018                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
11019                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
11020
11021         if (dev->flags & IFF_PROMISC)
11022                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11023
11024         uc_update = bnxt_uc_list_updated(bp);
11025
11026         if (dev->flags & IFF_BROADCAST)
11027                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
11028         if (dev->flags & IFF_ALLMULTI) {
11029                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
11030                 vnic->mc_list_count = 0;
11031         } else if (dev->flags & IFF_MULTICAST) {
11032                 mc_update = bnxt_mc_list_updated(bp, &mask);
11033         }
11034
11035         if (mask != vnic->rx_mask || uc_update || mc_update) {
11036                 vnic->rx_mask = mask;
11037
11038                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
11039                 bnxt_queue_sp_work(bp);
11040         }
11041 }
11042
11043 static int bnxt_cfg_rx_mode(struct bnxt *bp)
11044 {
11045         struct net_device *dev = bp->dev;
11046         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11047         struct hwrm_cfa_l2_filter_free_input *req;
11048         struct netdev_hw_addr *ha;
11049         int i, off = 0, rc;
11050         bool uc_update;
11051
11052         netif_addr_lock_bh(dev);
11053         uc_update = bnxt_uc_list_updated(bp);
11054         netif_addr_unlock_bh(dev);
11055
11056         if (!uc_update)
11057                 goto skip_uc;
11058
11059         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
11060         if (rc)
11061                 return rc;
11062         hwrm_req_hold(bp, req);
11063         for (i = 1; i < vnic->uc_filter_count; i++) {
11064                 req->l2_filter_id = vnic->fw_l2_filter_id[i];
11065
11066                 rc = hwrm_req_send(bp, req);
11067         }
11068         hwrm_req_drop(bp, req);
11069
11070         vnic->uc_filter_count = 1;
11071
11072         netif_addr_lock_bh(dev);
11073         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
11074                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11075         } else {
11076                 netdev_for_each_uc_addr(ha, dev) {
11077                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
11078                         off += ETH_ALEN;
11079                         vnic->uc_filter_count++;
11080                 }
11081         }
11082         netif_addr_unlock_bh(dev);
11083
11084         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
11085                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
11086                 if (rc) {
11087                         if (BNXT_VF(bp) && rc == -ENODEV) {
11088                                 if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
11089                                         netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n");
11090                                 else
11091                                         netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n");
11092                                 rc = 0;
11093                         } else {
11094                                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
11095                         }
11096                         vnic->uc_filter_count = i;
11097                         return rc;
11098                 }
11099         }
11100         if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
11101                 netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n");
11102
11103 skip_uc:
11104         if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) &&
11105             !bnxt_promisc_ok(bp))
11106                 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11107         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
11108         if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) {
11109                 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
11110                             rc);
11111                 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
11112                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
11113                 vnic->mc_list_count = 0;
11114                 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
11115         }
11116         if (rc)
11117                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
11118                            rc);
11119
11120         return rc;
11121 }
11122
11123 static bool bnxt_can_reserve_rings(struct bnxt *bp)
11124 {
11125 #ifdef CONFIG_BNXT_SRIOV
11126         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
11127                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11128
11129                 /* No minimum rings were provisioned by the PF.  Don't
11130                  * reserve rings by default when device is down.
11131                  */
11132                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
11133                         return true;
11134
11135                 if (!netif_running(bp->dev))
11136                         return false;
11137         }
11138 #endif
11139         return true;
11140 }
11141
11142 /* If the chip and firmware supports RFS */
11143 static bool bnxt_rfs_supported(struct bnxt *bp)
11144 {
11145         if (bp->flags & BNXT_FLAG_CHIP_P5) {
11146                 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
11147                         return true;
11148                 return false;
11149         }
11150         /* 212 firmware is broken for aRFS */
11151         if (BNXT_FW_MAJ(bp) == 212)
11152                 return false;
11153         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
11154                 return true;
11155         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
11156                 return true;
11157         return false;
11158 }
11159
11160 /* If runtime conditions support RFS */
11161 static bool bnxt_rfs_capable(struct bnxt *bp)
11162 {
11163 #ifdef CONFIG_RFS_ACCEL
11164         int vnics, max_vnics, max_rss_ctxs;
11165
11166         if (bp->flags & BNXT_FLAG_CHIP_P5)
11167                 return bnxt_rfs_supported(bp);
11168         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings)
11169                 return false;
11170
11171         vnics = 1 + bp->rx_nr_rings;
11172         max_vnics = bnxt_get_max_func_vnics(bp);
11173         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
11174
11175         /* RSS contexts not a limiting factor */
11176         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
11177                 max_rss_ctxs = max_vnics;
11178         if (vnics > max_vnics || vnics > max_rss_ctxs) {
11179                 if (bp->rx_nr_rings > 1)
11180                         netdev_warn(bp->dev,
11181                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
11182                                     min(max_rss_ctxs - 1, max_vnics - 1));
11183                 return false;
11184         }
11185
11186         if (!BNXT_NEW_RM(bp))
11187                 return true;
11188
11189         if (vnics == bp->hw_resc.resv_vnics)
11190                 return true;
11191
11192         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
11193         if (vnics <= bp->hw_resc.resv_vnics)
11194                 return true;
11195
11196         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
11197         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
11198         return false;
11199 #else
11200         return false;
11201 #endif
11202 }
11203
11204 static netdev_features_t bnxt_fix_features(struct net_device *dev,
11205                                            netdev_features_t features)
11206 {
11207         struct bnxt *bp = netdev_priv(dev);
11208         netdev_features_t vlan_features;
11209
11210         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
11211                 features &= ~NETIF_F_NTUPLE;
11212
11213         if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog)
11214                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11215
11216         if (!(features & NETIF_F_GRO))
11217                 features &= ~NETIF_F_GRO_HW;
11218
11219         if (features & NETIF_F_GRO_HW)
11220                 features &= ~NETIF_F_LRO;
11221
11222         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
11223          * turned on or off together.
11224          */
11225         vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
11226         if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
11227                 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
11228                         features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
11229                 else if (vlan_features)
11230                         features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
11231         }
11232 #ifdef CONFIG_BNXT_SRIOV
11233         if (BNXT_VF(bp) && bp->vf.vlan)
11234                 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
11235 #endif
11236         return features;
11237 }
11238
11239 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
11240 {
11241         struct bnxt *bp = netdev_priv(dev);
11242         u32 flags = bp->flags;
11243         u32 changes;
11244         int rc = 0;
11245         bool re_init = false;
11246         bool update_tpa = false;
11247
11248         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
11249         if (features & NETIF_F_GRO_HW)
11250                 flags |= BNXT_FLAG_GRO;
11251         else if (features & NETIF_F_LRO)
11252                 flags |= BNXT_FLAG_LRO;
11253
11254         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
11255                 flags &= ~BNXT_FLAG_TPA;
11256
11257         if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
11258                 flags |= BNXT_FLAG_STRIP_VLAN;
11259
11260         if (features & NETIF_F_NTUPLE)
11261                 flags |= BNXT_FLAG_RFS;
11262
11263         changes = flags ^ bp->flags;
11264         if (changes & BNXT_FLAG_TPA) {
11265                 update_tpa = true;
11266                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
11267                     (flags & BNXT_FLAG_TPA) == 0 ||
11268                     (bp->flags & BNXT_FLAG_CHIP_P5))
11269                         re_init = true;
11270         }
11271
11272         if (changes & ~BNXT_FLAG_TPA)
11273                 re_init = true;
11274
11275         if (flags != bp->flags) {
11276                 u32 old_flags = bp->flags;
11277
11278                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11279                         bp->flags = flags;
11280                         if (update_tpa)
11281                                 bnxt_set_ring_params(bp);
11282                         return rc;
11283                 }
11284
11285                 if (re_init) {
11286                         bnxt_close_nic(bp, false, false);
11287                         bp->flags = flags;
11288                         if (update_tpa)
11289                                 bnxt_set_ring_params(bp);
11290
11291                         return bnxt_open_nic(bp, false, false);
11292                 }
11293                 if (update_tpa) {
11294                         bp->flags = flags;
11295                         rc = bnxt_set_tpa(bp,
11296                                           (flags & BNXT_FLAG_TPA) ?
11297                                           true : false);
11298                         if (rc)
11299                                 bp->flags = old_flags;
11300                 }
11301         }
11302         return rc;
11303 }
11304
11305 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off,
11306                               u8 **nextp)
11307 {
11308         struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off);
11309         struct hop_jumbo_hdr *jhdr;
11310         int hdr_count = 0;
11311         u8 *nexthdr;
11312         int start;
11313
11314         /* Check that there are at most 2 IPv6 extension headers, no
11315          * fragment header, and each is <= 64 bytes.
11316          */
11317         start = nw_off + sizeof(*ip6h);
11318         nexthdr = &ip6h->nexthdr;
11319         while (ipv6_ext_hdr(*nexthdr)) {
11320                 struct ipv6_opt_hdr *hp;
11321                 int hdrlen;
11322
11323                 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE ||
11324                     *nexthdr == NEXTHDR_FRAGMENT)
11325                         return false;
11326                 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data,
11327                                           skb_headlen(skb), NULL);
11328                 if (!hp)
11329                         return false;
11330                 if (*nexthdr == NEXTHDR_AUTH)
11331                         hdrlen = ipv6_authlen(hp);
11332                 else
11333                         hdrlen = ipv6_optlen(hp);
11334
11335                 if (hdrlen > 64)
11336                         return false;
11337
11338                 /* The ext header may be a hop-by-hop header inserted for
11339                  * big TCP purposes. This will be removed before sending
11340                  * from NIC, so do not count it.
11341                  */
11342                 if (*nexthdr == NEXTHDR_HOP) {
11343                         if (likely(skb->len <= GRO_LEGACY_MAX_SIZE))
11344                                 goto increment_hdr;
11345
11346                         jhdr = (struct hop_jumbo_hdr *)hp;
11347                         if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 ||
11348                             jhdr->nexthdr != IPPROTO_TCP)
11349                                 goto increment_hdr;
11350
11351                         goto next_hdr;
11352                 }
11353 increment_hdr:
11354                 hdr_count++;
11355 next_hdr:
11356                 nexthdr = &hp->nexthdr;
11357                 start += hdrlen;
11358         }
11359         if (nextp) {
11360                 /* Caller will check inner protocol */
11361                 if (skb->encapsulation) {
11362                         *nextp = nexthdr;
11363                         return true;
11364                 }
11365                 *nextp = NULL;
11366         }
11367         /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */
11368         return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP;
11369 }
11370
11371 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */
11372 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb)
11373 {
11374         struct udphdr *uh = udp_hdr(skb);
11375         __be16 udp_port = uh->dest;
11376
11377         if (udp_port != bp->vxlan_port && udp_port != bp->nge_port)
11378                 return false;
11379         if (skb->inner_protocol_type == ENCAP_TYPE_ETHER) {
11380                 struct ethhdr *eh = inner_eth_hdr(skb);
11381
11382                 switch (eh->h_proto) {
11383                 case htons(ETH_P_IP):
11384                         return true;
11385                 case htons(ETH_P_IPV6):
11386                         return bnxt_exthdr_check(bp, skb,
11387                                                  skb_inner_network_offset(skb),
11388                                                  NULL);
11389                 }
11390         }
11391         return false;
11392 }
11393
11394 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto)
11395 {
11396         switch (l4_proto) {
11397         case IPPROTO_UDP:
11398                 return bnxt_udp_tunl_check(bp, skb);
11399         case IPPROTO_IPIP:
11400                 return true;
11401         case IPPROTO_GRE: {
11402                 switch (skb->inner_protocol) {
11403                 default:
11404                         return false;
11405                 case htons(ETH_P_IP):
11406                         return true;
11407                 case htons(ETH_P_IPV6):
11408                         fallthrough;
11409                 }
11410         }
11411         case IPPROTO_IPV6:
11412                 /* Check ext headers of inner ipv6 */
11413                 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
11414                                          NULL);
11415         }
11416         return false;
11417 }
11418
11419 static netdev_features_t bnxt_features_check(struct sk_buff *skb,
11420                                              struct net_device *dev,
11421                                              netdev_features_t features)
11422 {
11423         struct bnxt *bp = netdev_priv(dev);
11424         u8 *l4_proto;
11425
11426         features = vlan_features_check(skb, features);
11427         switch (vlan_get_protocol(skb)) {
11428         case htons(ETH_P_IP):
11429                 if (!skb->encapsulation)
11430                         return features;
11431                 l4_proto = &ip_hdr(skb)->protocol;
11432                 if (bnxt_tunl_check(bp, skb, *l4_proto))
11433                         return features;
11434                 break;
11435         case htons(ETH_P_IPV6):
11436                 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb),
11437                                        &l4_proto))
11438                         break;
11439                 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto))
11440                         return features;
11441                 break;
11442         }
11443         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
11444 }
11445
11446 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
11447                          u32 *reg_buf)
11448 {
11449         struct hwrm_dbg_read_direct_output *resp;
11450         struct hwrm_dbg_read_direct_input *req;
11451         __le32 *dbg_reg_buf;
11452         dma_addr_t mapping;
11453         int rc, i;
11454
11455         rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT);
11456         if (rc)
11457                 return rc;
11458
11459         dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4,
11460                                          &mapping);
11461         if (!dbg_reg_buf) {
11462                 rc = -ENOMEM;
11463                 goto dbg_rd_reg_exit;
11464         }
11465
11466         req->host_dest_addr = cpu_to_le64(mapping);
11467
11468         resp = hwrm_req_hold(bp, req);
11469         req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
11470         req->read_len32 = cpu_to_le32(num_words);
11471
11472         rc = hwrm_req_send(bp, req);
11473         if (rc || resp->error_code) {
11474                 rc = -EIO;
11475                 goto dbg_rd_reg_exit;
11476         }
11477         for (i = 0; i < num_words; i++)
11478                 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
11479
11480 dbg_rd_reg_exit:
11481         hwrm_req_drop(bp, req);
11482         return rc;
11483 }
11484
11485 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
11486                                        u32 ring_id, u32 *prod, u32 *cons)
11487 {
11488         struct hwrm_dbg_ring_info_get_output *resp;
11489         struct hwrm_dbg_ring_info_get_input *req;
11490         int rc;
11491
11492         rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET);
11493         if (rc)
11494                 return rc;
11495
11496         req->ring_type = ring_type;
11497         req->fw_ring_id = cpu_to_le32(ring_id);
11498         resp = hwrm_req_hold(bp, req);
11499         rc = hwrm_req_send(bp, req);
11500         if (!rc) {
11501                 *prod = le32_to_cpu(resp->producer_index);
11502                 *cons = le32_to_cpu(resp->consumer_index);
11503         }
11504         hwrm_req_drop(bp, req);
11505         return rc;
11506 }
11507
11508 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
11509 {
11510         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
11511         int i = bnapi->index;
11512
11513         if (!txr)
11514                 return;
11515
11516         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
11517                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
11518                     txr->tx_cons);
11519 }
11520
11521 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
11522 {
11523         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
11524         int i = bnapi->index;
11525
11526         if (!rxr)
11527                 return;
11528
11529         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
11530                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
11531                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
11532                     rxr->rx_sw_agg_prod);
11533 }
11534
11535 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
11536 {
11537         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
11538         int i = bnapi->index;
11539
11540         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
11541                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
11542 }
11543
11544 static void bnxt_dbg_dump_states(struct bnxt *bp)
11545 {
11546         int i;
11547         struct bnxt_napi *bnapi;
11548
11549         for (i = 0; i < bp->cp_nr_rings; i++) {
11550                 bnapi = bp->bnapi[i];
11551                 if (netif_msg_drv(bp)) {
11552                         bnxt_dump_tx_sw_state(bnapi);
11553                         bnxt_dump_rx_sw_state(bnapi);
11554                         bnxt_dump_cp_sw_state(bnapi);
11555                 }
11556         }
11557 }
11558
11559 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
11560 {
11561         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
11562         struct hwrm_ring_reset_input *req;
11563         struct bnxt_napi *bnapi = rxr->bnapi;
11564         struct bnxt_cp_ring_info *cpr;
11565         u16 cp_ring_id;
11566         int rc;
11567
11568         rc = hwrm_req_init(bp, req, HWRM_RING_RESET);
11569         if (rc)
11570                 return rc;
11571
11572         cpr = &bnapi->cp_ring;
11573         cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
11574         req->cmpl_ring = cpu_to_le16(cp_ring_id);
11575         req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
11576         req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
11577         return hwrm_req_send_silent(bp, req);
11578 }
11579
11580 static void bnxt_reset_task(struct bnxt *bp, bool silent)
11581 {
11582         if (!silent)
11583                 bnxt_dbg_dump_states(bp);
11584         if (netif_running(bp->dev)) {
11585                 int rc;
11586
11587                 if (silent) {
11588                         bnxt_close_nic(bp, false, false);
11589                         bnxt_open_nic(bp, false, false);
11590                 } else {
11591                         bnxt_ulp_stop(bp);
11592                         bnxt_close_nic(bp, true, false);
11593                         rc = bnxt_open_nic(bp, true, false);
11594                         bnxt_ulp_start(bp, rc);
11595                 }
11596         }
11597 }
11598
11599 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
11600 {
11601         struct bnxt *bp = netdev_priv(dev);
11602
11603         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
11604         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
11605         bnxt_queue_sp_work(bp);
11606 }
11607
11608 static void bnxt_fw_health_check(struct bnxt *bp)
11609 {
11610         struct bnxt_fw_health *fw_health = bp->fw_health;
11611         struct pci_dev *pdev = bp->pdev;
11612         u32 val;
11613
11614         if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11615                 return;
11616
11617         /* Make sure it is enabled before checking the tmr_counter. */
11618         smp_rmb();
11619         if (fw_health->tmr_counter) {
11620                 fw_health->tmr_counter--;
11621                 return;
11622         }
11623
11624         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
11625         if (val == fw_health->last_fw_heartbeat && pci_device_is_present(pdev)) {
11626                 fw_health->arrests++;
11627                 goto fw_reset;
11628         }
11629
11630         fw_health->last_fw_heartbeat = val;
11631
11632         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
11633         if (val != fw_health->last_fw_reset_cnt && pci_device_is_present(pdev)) {
11634                 fw_health->discoveries++;
11635                 goto fw_reset;
11636         }
11637
11638         fw_health->tmr_counter = fw_health->tmr_multiplier;
11639         return;
11640
11641 fw_reset:
11642         set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
11643         bnxt_queue_sp_work(bp);
11644 }
11645
11646 static void bnxt_timer(struct timer_list *t)
11647 {
11648         struct bnxt *bp = from_timer(bp, t, timer);
11649         struct net_device *dev = bp->dev;
11650
11651         if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
11652                 return;
11653
11654         if (atomic_read(&bp->intr_sem) != 0)
11655                 goto bnxt_restart_timer;
11656
11657         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
11658                 bnxt_fw_health_check(bp);
11659
11660         if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks) {
11661                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
11662                 bnxt_queue_sp_work(bp);
11663         }
11664
11665         if (bnxt_tc_flower_enabled(bp)) {
11666                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
11667                 bnxt_queue_sp_work(bp);
11668         }
11669
11670 #ifdef CONFIG_RFS_ACCEL
11671         if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) {
11672                 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
11673                 bnxt_queue_sp_work(bp);
11674         }
11675 #endif /*CONFIG_RFS_ACCEL*/
11676
11677         if (bp->link_info.phy_retry) {
11678                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
11679                         bp->link_info.phy_retry = false;
11680                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
11681                 } else {
11682                         set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
11683                         bnxt_queue_sp_work(bp);
11684                 }
11685         }
11686
11687         if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) {
11688                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
11689                 bnxt_queue_sp_work(bp);
11690         }
11691
11692         if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
11693             netif_carrier_ok(dev)) {
11694                 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
11695                 bnxt_queue_sp_work(bp);
11696         }
11697 bnxt_restart_timer:
11698         mod_timer(&bp->timer, jiffies + bp->current_interval);
11699 }
11700
11701 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
11702 {
11703         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
11704          * set.  If the device is being closed, bnxt_close() may be holding
11705          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
11706          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
11707          */
11708         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11709         rtnl_lock();
11710 }
11711
11712 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
11713 {
11714         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11715         rtnl_unlock();
11716 }
11717
11718 /* Only called from bnxt_sp_task() */
11719 static void bnxt_reset(struct bnxt *bp, bool silent)
11720 {
11721         bnxt_rtnl_lock_sp(bp);
11722         if (test_bit(BNXT_STATE_OPEN, &bp->state))
11723                 bnxt_reset_task(bp, silent);
11724         bnxt_rtnl_unlock_sp(bp);
11725 }
11726
11727 /* Only called from bnxt_sp_task() */
11728 static void bnxt_rx_ring_reset(struct bnxt *bp)
11729 {
11730         int i;
11731
11732         bnxt_rtnl_lock_sp(bp);
11733         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11734                 bnxt_rtnl_unlock_sp(bp);
11735                 return;
11736         }
11737         /* Disable and flush TPA before resetting the RX ring */
11738         if (bp->flags & BNXT_FLAG_TPA)
11739                 bnxt_set_tpa(bp, false);
11740         for (i = 0; i < bp->rx_nr_rings; i++) {
11741                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
11742                 struct bnxt_cp_ring_info *cpr;
11743                 int rc;
11744
11745                 if (!rxr->bnapi->in_reset)
11746                         continue;
11747
11748                 rc = bnxt_hwrm_rx_ring_reset(bp, i);
11749                 if (rc) {
11750                         if (rc == -EINVAL || rc == -EOPNOTSUPP)
11751                                 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
11752                         else
11753                                 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
11754                                             rc);
11755                         bnxt_reset_task(bp, true);
11756                         break;
11757                 }
11758                 bnxt_free_one_rx_ring_skbs(bp, i);
11759                 rxr->rx_prod = 0;
11760                 rxr->rx_agg_prod = 0;
11761                 rxr->rx_sw_agg_prod = 0;
11762                 rxr->rx_next_cons = 0;
11763                 rxr->bnapi->in_reset = false;
11764                 bnxt_alloc_one_rx_ring(bp, i);
11765                 cpr = &rxr->bnapi->cp_ring;
11766                 cpr->sw_stats.rx.rx_resets++;
11767                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
11768                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
11769                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
11770         }
11771         if (bp->flags & BNXT_FLAG_TPA)
11772                 bnxt_set_tpa(bp, true);
11773         bnxt_rtnl_unlock_sp(bp);
11774 }
11775
11776 static void bnxt_fw_reset_close(struct bnxt *bp)
11777 {
11778         bnxt_ulp_stop(bp);
11779         /* When firmware is in fatal state, quiesce device and disable
11780          * bus master to prevent any potential bad DMAs before freeing
11781          * kernel memory.
11782          */
11783         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11784                 u16 val = 0;
11785
11786                 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
11787                 if (val == 0xffff)
11788                         bp->fw_reset_min_dsecs = 0;
11789                 bnxt_tx_disable(bp);
11790                 bnxt_disable_napi(bp);
11791                 bnxt_disable_int_sync(bp);
11792                 bnxt_free_irq(bp);
11793                 bnxt_clear_int_mode(bp);
11794                 pci_disable_device(bp->pdev);
11795         }
11796         __bnxt_close_nic(bp, true, false);
11797         bnxt_vf_reps_free(bp);
11798         bnxt_clear_int_mode(bp);
11799         bnxt_hwrm_func_drv_unrgtr(bp);
11800         if (pci_is_enabled(bp->pdev))
11801                 pci_disable_device(bp->pdev);
11802         bnxt_free_ctx_mem(bp);
11803         kfree(bp->ctx);
11804         bp->ctx = NULL;
11805 }
11806
11807 static bool is_bnxt_fw_ok(struct bnxt *bp)
11808 {
11809         struct bnxt_fw_health *fw_health = bp->fw_health;
11810         bool no_heartbeat = false, has_reset = false;
11811         u32 val;
11812
11813         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
11814         if (val == fw_health->last_fw_heartbeat)
11815                 no_heartbeat = true;
11816
11817         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
11818         if (val != fw_health->last_fw_reset_cnt)
11819                 has_reset = true;
11820
11821         if (!no_heartbeat && has_reset)
11822                 return true;
11823
11824         return false;
11825 }
11826
11827 /* rtnl_lock is acquired before calling this function */
11828 static void bnxt_force_fw_reset(struct bnxt *bp)
11829 {
11830         struct bnxt_fw_health *fw_health = bp->fw_health;
11831         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
11832         u32 wait_dsecs;
11833
11834         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
11835             test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11836                 return;
11837
11838         if (ptp) {
11839                 spin_lock_bh(&ptp->ptp_lock);
11840                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11841                 spin_unlock_bh(&ptp->ptp_lock);
11842         } else {
11843                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11844         }
11845         bnxt_fw_reset_close(bp);
11846         wait_dsecs = fw_health->master_func_wait_dsecs;
11847         if (fw_health->primary) {
11848                 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
11849                         wait_dsecs = 0;
11850                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11851         } else {
11852                 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
11853                 wait_dsecs = fw_health->normal_func_wait_dsecs;
11854                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11855         }
11856
11857         bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
11858         bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
11859         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11860 }
11861
11862 void bnxt_fw_exception(struct bnxt *bp)
11863 {
11864         netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
11865         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11866         bnxt_rtnl_lock_sp(bp);
11867         bnxt_force_fw_reset(bp);
11868         bnxt_rtnl_unlock_sp(bp);
11869 }
11870
11871 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
11872  * < 0 on error.
11873  */
11874 static int bnxt_get_registered_vfs(struct bnxt *bp)
11875 {
11876 #ifdef CONFIG_BNXT_SRIOV
11877         int rc;
11878
11879         if (!BNXT_PF(bp))
11880                 return 0;
11881
11882         rc = bnxt_hwrm_func_qcfg(bp);
11883         if (rc) {
11884                 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
11885                 return rc;
11886         }
11887         if (bp->pf.registered_vfs)
11888                 return bp->pf.registered_vfs;
11889         if (bp->sriov_cfg)
11890                 return 1;
11891 #endif
11892         return 0;
11893 }
11894
11895 void bnxt_fw_reset(struct bnxt *bp)
11896 {
11897         bnxt_rtnl_lock_sp(bp);
11898         if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
11899             !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11900                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
11901                 int n = 0, tmo;
11902
11903                 if (ptp) {
11904                         spin_lock_bh(&ptp->ptp_lock);
11905                         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11906                         spin_unlock_bh(&ptp->ptp_lock);
11907                 } else {
11908                         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11909                 }
11910                 if (bp->pf.active_vfs &&
11911                     !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
11912                         n = bnxt_get_registered_vfs(bp);
11913                 if (n < 0) {
11914                         netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
11915                                    n);
11916                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11917                         dev_close(bp->dev);
11918                         goto fw_reset_exit;
11919                 } else if (n > 0) {
11920                         u16 vf_tmo_dsecs = n * 10;
11921
11922                         if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
11923                                 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
11924                         bp->fw_reset_state =
11925                                 BNXT_FW_RESET_STATE_POLL_VF;
11926                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11927                         goto fw_reset_exit;
11928                 }
11929                 bnxt_fw_reset_close(bp);
11930                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11931                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11932                         tmo = HZ / 10;
11933                 } else {
11934                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11935                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
11936                 }
11937                 bnxt_queue_fw_reset_work(bp, tmo);
11938         }
11939 fw_reset_exit:
11940         bnxt_rtnl_unlock_sp(bp);
11941 }
11942
11943 static void bnxt_chk_missed_irq(struct bnxt *bp)
11944 {
11945         int i;
11946
11947         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
11948                 return;
11949
11950         for (i = 0; i < bp->cp_nr_rings; i++) {
11951                 struct bnxt_napi *bnapi = bp->bnapi[i];
11952                 struct bnxt_cp_ring_info *cpr;
11953                 u32 fw_ring_id;
11954                 int j;
11955
11956                 if (!bnapi)
11957                         continue;
11958
11959                 cpr = &bnapi->cp_ring;
11960                 for (j = 0; j < 2; j++) {
11961                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
11962                         u32 val[2];
11963
11964                         if (!cpr2 || cpr2->has_more_work ||
11965                             !bnxt_has_work(bp, cpr2))
11966                                 continue;
11967
11968                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
11969                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
11970                                 continue;
11971                         }
11972                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
11973                         bnxt_dbg_hwrm_ring_info_get(bp,
11974                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
11975                                 fw_ring_id, &val[0], &val[1]);
11976                         cpr->sw_stats.cmn.missed_irqs++;
11977                 }
11978         }
11979 }
11980
11981 static void bnxt_cfg_ntp_filters(struct bnxt *);
11982
11983 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
11984 {
11985         struct bnxt_link_info *link_info = &bp->link_info;
11986
11987         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
11988                 link_info->autoneg = BNXT_AUTONEG_SPEED;
11989                 if (bp->hwrm_spec_code >= 0x10201) {
11990                         if (link_info->auto_pause_setting &
11991                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
11992                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
11993                 } else {
11994                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
11995                 }
11996                 link_info->advertising = link_info->auto_link_speeds;
11997                 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
11998         } else {
11999                 link_info->req_link_speed = link_info->force_link_speed;
12000                 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
12001                 if (link_info->force_pam4_link_speed) {
12002                         link_info->req_link_speed =
12003                                 link_info->force_pam4_link_speed;
12004                         link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
12005                 }
12006                 link_info->req_duplex = link_info->duplex_setting;
12007         }
12008         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
12009                 link_info->req_flow_ctrl =
12010                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
12011         else
12012                 link_info->req_flow_ctrl = link_info->force_pause_setting;
12013 }
12014
12015 static void bnxt_fw_echo_reply(struct bnxt *bp)
12016 {
12017         struct bnxt_fw_health *fw_health = bp->fw_health;
12018         struct hwrm_func_echo_response_input *req;
12019         int rc;
12020
12021         rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE);
12022         if (rc)
12023                 return;
12024         req->event_data1 = cpu_to_le32(fw_health->echo_req_data1);
12025         req->event_data2 = cpu_to_le32(fw_health->echo_req_data2);
12026         hwrm_req_send(bp, req);
12027 }
12028
12029 static void bnxt_sp_task(struct work_struct *work)
12030 {
12031         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
12032
12033         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12034         smp_mb__after_atomic();
12035         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
12036                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12037                 return;
12038         }
12039
12040         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
12041                 bnxt_cfg_rx_mode(bp);
12042
12043         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
12044                 bnxt_cfg_ntp_filters(bp);
12045         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
12046                 bnxt_hwrm_exec_fwd_req(bp);
12047         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
12048                 bnxt_hwrm_port_qstats(bp, 0);
12049                 bnxt_hwrm_port_qstats_ext(bp, 0);
12050                 bnxt_accumulate_all_stats(bp);
12051         }
12052
12053         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
12054                 int rc;
12055
12056                 mutex_lock(&bp->link_lock);
12057                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
12058                                        &bp->sp_event))
12059                         bnxt_hwrm_phy_qcaps(bp);
12060
12061                 rc = bnxt_update_link(bp, true);
12062                 if (rc)
12063                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
12064                                    rc);
12065
12066                 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
12067                                        &bp->sp_event))
12068                         bnxt_init_ethtool_link_settings(bp);
12069                 mutex_unlock(&bp->link_lock);
12070         }
12071         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
12072                 int rc;
12073
12074                 mutex_lock(&bp->link_lock);
12075                 rc = bnxt_update_phy_setting(bp);
12076                 mutex_unlock(&bp->link_lock);
12077                 if (rc) {
12078                         netdev_warn(bp->dev, "update phy settings retry failed\n");
12079                 } else {
12080                         bp->link_info.phy_retry = false;
12081                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
12082                 }
12083         }
12084         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
12085                 mutex_lock(&bp->link_lock);
12086                 bnxt_get_port_module_status(bp);
12087                 mutex_unlock(&bp->link_lock);
12088         }
12089
12090         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
12091                 bnxt_tc_flow_stats_work(bp);
12092
12093         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
12094                 bnxt_chk_missed_irq(bp);
12095
12096         if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event))
12097                 bnxt_fw_echo_reply(bp);
12098
12099         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
12100          * must be the last functions to be called before exiting.
12101          */
12102         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
12103                 bnxt_reset(bp, false);
12104
12105         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
12106                 bnxt_reset(bp, true);
12107
12108         if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
12109                 bnxt_rx_ring_reset(bp);
12110
12111         if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) {
12112                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) ||
12113                     test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state))
12114                         bnxt_devlink_health_fw_report(bp);
12115                 else
12116                         bnxt_fw_reset(bp);
12117         }
12118
12119         if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
12120                 if (!is_bnxt_fw_ok(bp))
12121                         bnxt_devlink_health_fw_report(bp);
12122         }
12123
12124         smp_mb__before_atomic();
12125         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12126 }
12127
12128 /* Under rtnl_lock */
12129 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
12130                      int tx_xdp)
12131 {
12132         int max_rx, max_tx, tx_sets = 1;
12133         int tx_rings_needed, stats;
12134         int rx_rings = rx;
12135         int cp, vnics, rc;
12136
12137         if (tcs)
12138                 tx_sets = tcs;
12139
12140         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
12141         if (rc)
12142                 return rc;
12143
12144         if (max_rx < rx)
12145                 return -ENOMEM;
12146
12147         tx_rings_needed = tx * tx_sets + tx_xdp;
12148         if (max_tx < tx_rings_needed)
12149                 return -ENOMEM;
12150
12151         vnics = 1;
12152         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
12153                 vnics += rx_rings;
12154
12155         if (bp->flags & BNXT_FLAG_AGG_RINGS)
12156                 rx_rings <<= 1;
12157         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
12158         stats = cp;
12159         if (BNXT_NEW_RM(bp)) {
12160                 cp += bnxt_get_ulp_msix_num(bp);
12161                 stats += bnxt_get_ulp_stat_ctxs(bp);
12162         }
12163         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
12164                                      stats, vnics);
12165 }
12166
12167 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
12168 {
12169         if (bp->bar2) {
12170                 pci_iounmap(pdev, bp->bar2);
12171                 bp->bar2 = NULL;
12172         }
12173
12174         if (bp->bar1) {
12175                 pci_iounmap(pdev, bp->bar1);
12176                 bp->bar1 = NULL;
12177         }
12178
12179         if (bp->bar0) {
12180                 pci_iounmap(pdev, bp->bar0);
12181                 bp->bar0 = NULL;
12182         }
12183 }
12184
12185 static void bnxt_cleanup_pci(struct bnxt *bp)
12186 {
12187         bnxt_unmap_bars(bp, bp->pdev);
12188         pci_release_regions(bp->pdev);
12189         if (pci_is_enabled(bp->pdev))
12190                 pci_disable_device(bp->pdev);
12191 }
12192
12193 static void bnxt_init_dflt_coal(struct bnxt *bp)
12194 {
12195         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
12196         struct bnxt_coal *coal;
12197         u16 flags = 0;
12198
12199         if (coal_cap->cmpl_params &
12200             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
12201                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
12202
12203         /* Tick values in micro seconds.
12204          * 1 coal_buf x bufs_per_record = 1 completion record.
12205          */
12206         coal = &bp->rx_coal;
12207         coal->coal_ticks = 10;
12208         coal->coal_bufs = 30;
12209         coal->coal_ticks_irq = 1;
12210         coal->coal_bufs_irq = 2;
12211         coal->idle_thresh = 50;
12212         coal->bufs_per_record = 2;
12213         coal->budget = 64;              /* NAPI budget */
12214         coal->flags = flags;
12215
12216         coal = &bp->tx_coal;
12217         coal->coal_ticks = 28;
12218         coal->coal_bufs = 30;
12219         coal->coal_ticks_irq = 2;
12220         coal->coal_bufs_irq = 2;
12221         coal->bufs_per_record = 1;
12222         coal->flags = flags;
12223
12224         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
12225 }
12226
12227 static int bnxt_fw_init_one_p1(struct bnxt *bp)
12228 {
12229         int rc;
12230
12231         bp->fw_cap = 0;
12232         rc = bnxt_hwrm_ver_get(bp);
12233         bnxt_try_map_fw_health_reg(bp);
12234         if (rc) {
12235                 rc = bnxt_try_recover_fw(bp);
12236                 if (rc)
12237                         return rc;
12238                 rc = bnxt_hwrm_ver_get(bp);
12239                 if (rc)
12240                         return rc;
12241         }
12242
12243         bnxt_nvm_cfg_ver_get(bp);
12244
12245         rc = bnxt_hwrm_func_reset(bp);
12246         if (rc)
12247                 return -ENODEV;
12248
12249         bnxt_hwrm_fw_set_time(bp);
12250         return 0;
12251 }
12252
12253 static int bnxt_fw_init_one_p2(struct bnxt *bp)
12254 {
12255         int rc;
12256
12257         /* Get the MAX capabilities for this function */
12258         rc = bnxt_hwrm_func_qcaps(bp);
12259         if (rc) {
12260                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
12261                            rc);
12262                 return -ENODEV;
12263         }
12264
12265         rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
12266         if (rc)
12267                 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
12268                             rc);
12269
12270         if (bnxt_alloc_fw_health(bp)) {
12271                 netdev_warn(bp->dev, "no memory for firmware error recovery\n");
12272         } else {
12273                 rc = bnxt_hwrm_error_recovery_qcfg(bp);
12274                 if (rc)
12275                         netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
12276                                     rc);
12277         }
12278
12279         rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
12280         if (rc)
12281                 return -ENODEV;
12282
12283         bnxt_hwrm_func_qcfg(bp);
12284         bnxt_hwrm_vnic_qcaps(bp);
12285         bnxt_hwrm_port_led_qcaps(bp);
12286         bnxt_ethtool_init(bp);
12287         if (bp->fw_cap & BNXT_FW_CAP_PTP)
12288                 __bnxt_hwrm_ptp_qcfg(bp);
12289         bnxt_dcb_init(bp);
12290         return 0;
12291 }
12292
12293 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
12294 {
12295         bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
12296         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
12297                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
12298                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
12299                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
12300         if (bp->fw_cap & BNXT_FW_CAP_RSS_HASH_TYPE_DELTA)
12301                 bp->rss_hash_delta = bp->rss_hash_cfg;
12302         if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
12303                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
12304                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
12305                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
12306         }
12307 }
12308
12309 static void bnxt_set_dflt_rfs(struct bnxt *bp)
12310 {
12311         struct net_device *dev = bp->dev;
12312
12313         dev->hw_features &= ~NETIF_F_NTUPLE;
12314         dev->features &= ~NETIF_F_NTUPLE;
12315         bp->flags &= ~BNXT_FLAG_RFS;
12316         if (bnxt_rfs_supported(bp)) {
12317                 dev->hw_features |= NETIF_F_NTUPLE;
12318                 if (bnxt_rfs_capable(bp)) {
12319                         bp->flags |= BNXT_FLAG_RFS;
12320                         dev->features |= NETIF_F_NTUPLE;
12321                 }
12322         }
12323 }
12324
12325 static void bnxt_fw_init_one_p3(struct bnxt *bp)
12326 {
12327         struct pci_dev *pdev = bp->pdev;
12328
12329         bnxt_set_dflt_rss_hash_type(bp);
12330         bnxt_set_dflt_rfs(bp);
12331
12332         bnxt_get_wol_settings(bp);
12333         if (bp->flags & BNXT_FLAG_WOL_CAP)
12334                 device_set_wakeup_enable(&pdev->dev, bp->wol);
12335         else
12336                 device_set_wakeup_capable(&pdev->dev, false);
12337
12338         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
12339         bnxt_hwrm_coal_params_qcaps(bp);
12340 }
12341
12342 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt);
12343
12344 int bnxt_fw_init_one(struct bnxt *bp)
12345 {
12346         int rc;
12347
12348         rc = bnxt_fw_init_one_p1(bp);
12349         if (rc) {
12350                 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
12351                 return rc;
12352         }
12353         rc = bnxt_fw_init_one_p2(bp);
12354         if (rc) {
12355                 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
12356                 return rc;
12357         }
12358         rc = bnxt_probe_phy(bp, false);
12359         if (rc)
12360                 return rc;
12361         rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
12362         if (rc)
12363                 return rc;
12364
12365         bnxt_fw_init_one_p3(bp);
12366         return 0;
12367 }
12368
12369 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
12370 {
12371         struct bnxt_fw_health *fw_health = bp->fw_health;
12372         u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
12373         u32 val = fw_health->fw_reset_seq_vals[reg_idx];
12374         u32 reg_type, reg_off, delay_msecs;
12375
12376         delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
12377         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
12378         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
12379         switch (reg_type) {
12380         case BNXT_FW_HEALTH_REG_TYPE_CFG:
12381                 pci_write_config_dword(bp->pdev, reg_off, val);
12382                 break;
12383         case BNXT_FW_HEALTH_REG_TYPE_GRC:
12384                 writel(reg_off & BNXT_GRC_BASE_MASK,
12385                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
12386                 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
12387                 fallthrough;
12388         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
12389                 writel(val, bp->bar0 + reg_off);
12390                 break;
12391         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
12392                 writel(val, bp->bar1 + reg_off);
12393                 break;
12394         }
12395         if (delay_msecs) {
12396                 pci_read_config_dword(bp->pdev, 0, &val);
12397                 msleep(delay_msecs);
12398         }
12399 }
12400
12401 bool bnxt_hwrm_reset_permitted(struct bnxt *bp)
12402 {
12403         struct hwrm_func_qcfg_output *resp;
12404         struct hwrm_func_qcfg_input *req;
12405         bool result = true; /* firmware will enforce if unknown */
12406
12407         if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF)
12408                 return result;
12409
12410         if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG))
12411                 return result;
12412
12413         req->fid = cpu_to_le16(0xffff);
12414         resp = hwrm_req_hold(bp, req);
12415         if (!hwrm_req_send(bp, req))
12416                 result = !!(le16_to_cpu(resp->flags) &
12417                             FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED);
12418         hwrm_req_drop(bp, req);
12419         return result;
12420 }
12421
12422 static void bnxt_reset_all(struct bnxt *bp)
12423 {
12424         struct bnxt_fw_health *fw_health = bp->fw_health;
12425         int i, rc;
12426
12427         if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
12428                 bnxt_fw_reset_via_optee(bp);
12429                 bp->fw_reset_timestamp = jiffies;
12430                 return;
12431         }
12432
12433         if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
12434                 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
12435                         bnxt_fw_reset_writel(bp, i);
12436         } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
12437                 struct hwrm_fw_reset_input *req;
12438
12439                 rc = hwrm_req_init(bp, req, HWRM_FW_RESET);
12440                 if (!rc) {
12441                         req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG);
12442                         req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
12443                         req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
12444                         req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
12445                         rc = hwrm_req_send(bp, req);
12446                 }
12447                 if (rc != -ENODEV)
12448                         netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
12449         }
12450         bp->fw_reset_timestamp = jiffies;
12451 }
12452
12453 static bool bnxt_fw_reset_timeout(struct bnxt *bp)
12454 {
12455         return time_after(jiffies, bp->fw_reset_timestamp +
12456                           (bp->fw_reset_max_dsecs * HZ / 10));
12457 }
12458
12459 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc)
12460 {
12461         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12462         if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) {
12463                 bnxt_ulp_start(bp, rc);
12464                 bnxt_dl_health_fw_status_update(bp, false);
12465         }
12466         bp->fw_reset_state = 0;
12467         dev_close(bp->dev);
12468 }
12469
12470 static void bnxt_fw_reset_task(struct work_struct *work)
12471 {
12472         struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
12473         int rc = 0;
12474
12475         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
12476                 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
12477                 return;
12478         }
12479
12480         switch (bp->fw_reset_state) {
12481         case BNXT_FW_RESET_STATE_POLL_VF: {
12482                 int n = bnxt_get_registered_vfs(bp);
12483                 int tmo;
12484
12485                 if (n < 0) {
12486                         netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
12487                                    n, jiffies_to_msecs(jiffies -
12488                                    bp->fw_reset_timestamp));
12489                         goto fw_reset_abort;
12490                 } else if (n > 0) {
12491                         if (bnxt_fw_reset_timeout(bp)) {
12492                                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12493                                 bp->fw_reset_state = 0;
12494                                 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
12495                                            n);
12496                                 return;
12497                         }
12498                         bnxt_queue_fw_reset_work(bp, HZ / 10);
12499                         return;
12500                 }
12501                 bp->fw_reset_timestamp = jiffies;
12502                 rtnl_lock();
12503                 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
12504                         bnxt_fw_reset_abort(bp, rc);
12505                         rtnl_unlock();
12506                         return;
12507                 }
12508                 bnxt_fw_reset_close(bp);
12509                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
12510                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
12511                         tmo = HZ / 10;
12512                 } else {
12513                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12514                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
12515                 }
12516                 rtnl_unlock();
12517                 bnxt_queue_fw_reset_work(bp, tmo);
12518                 return;
12519         }
12520         case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
12521                 u32 val;
12522
12523                 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
12524                 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
12525                     !bnxt_fw_reset_timeout(bp)) {
12526                         bnxt_queue_fw_reset_work(bp, HZ / 5);
12527                         return;
12528                 }
12529
12530                 if (!bp->fw_health->primary) {
12531                         u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
12532
12533                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12534                         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
12535                         return;
12536                 }
12537                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
12538         }
12539                 fallthrough;
12540         case BNXT_FW_RESET_STATE_RESET_FW:
12541                 bnxt_reset_all(bp);
12542                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12543                 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
12544                 return;
12545         case BNXT_FW_RESET_STATE_ENABLE_DEV:
12546                 bnxt_inv_fw_health_reg(bp);
12547                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) &&
12548                     !bp->fw_reset_min_dsecs) {
12549                         u16 val;
12550
12551                         pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
12552                         if (val == 0xffff) {
12553                                 if (bnxt_fw_reset_timeout(bp)) {
12554                                         netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n");
12555                                         rc = -ETIMEDOUT;
12556                                         goto fw_reset_abort;
12557                                 }
12558                                 bnxt_queue_fw_reset_work(bp, HZ / 1000);
12559                                 return;
12560                         }
12561                 }
12562                 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
12563                 clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
12564                 if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) &&
12565                     !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state))
12566                         bnxt_dl_remote_reload(bp);
12567                 if (pci_enable_device(bp->pdev)) {
12568                         netdev_err(bp->dev, "Cannot re-enable PCI device\n");
12569                         rc = -ENODEV;
12570                         goto fw_reset_abort;
12571                 }
12572                 pci_set_master(bp->pdev);
12573                 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
12574                 fallthrough;
12575         case BNXT_FW_RESET_STATE_POLL_FW:
12576                 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
12577                 rc = bnxt_hwrm_poll(bp);
12578                 if (rc) {
12579                         if (bnxt_fw_reset_timeout(bp)) {
12580                                 netdev_err(bp->dev, "Firmware reset aborted\n");
12581                                 goto fw_reset_abort_status;
12582                         }
12583                         bnxt_queue_fw_reset_work(bp, HZ / 5);
12584                         return;
12585                 }
12586                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
12587                 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
12588                 fallthrough;
12589         case BNXT_FW_RESET_STATE_OPENING:
12590                 while (!rtnl_trylock()) {
12591                         bnxt_queue_fw_reset_work(bp, HZ / 10);
12592                         return;
12593                 }
12594                 rc = bnxt_open(bp->dev);
12595                 if (rc) {
12596                         netdev_err(bp->dev, "bnxt_open() failed during FW reset\n");
12597                         bnxt_fw_reset_abort(bp, rc);
12598                         rtnl_unlock();
12599                         return;
12600                 }
12601
12602                 if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) &&
12603                     bp->fw_health->enabled) {
12604                         bp->fw_health->last_fw_reset_cnt =
12605                                 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
12606                 }
12607                 bp->fw_reset_state = 0;
12608                 /* Make sure fw_reset_state is 0 before clearing the flag */
12609                 smp_mb__before_atomic();
12610                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12611                 bnxt_ulp_start(bp, 0);
12612                 bnxt_reenable_sriov(bp);
12613                 bnxt_vf_reps_alloc(bp);
12614                 bnxt_vf_reps_open(bp);
12615                 bnxt_ptp_reapply_pps(bp);
12616                 clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state);
12617                 if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) {
12618                         bnxt_dl_health_fw_recovery_done(bp);
12619                         bnxt_dl_health_fw_status_update(bp, true);
12620                 }
12621                 rtnl_unlock();
12622                 break;
12623         }
12624         return;
12625
12626 fw_reset_abort_status:
12627         if (bp->fw_health->status_reliable ||
12628             (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
12629                 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
12630
12631                 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
12632         }
12633 fw_reset_abort:
12634         rtnl_lock();
12635         bnxt_fw_reset_abort(bp, rc);
12636         rtnl_unlock();
12637 }
12638
12639 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
12640 {
12641         int rc;
12642         struct bnxt *bp = netdev_priv(dev);
12643
12644         SET_NETDEV_DEV(dev, &pdev->dev);
12645
12646         /* enable device (incl. PCI PM wakeup), and bus-mastering */
12647         rc = pci_enable_device(pdev);
12648         if (rc) {
12649                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
12650                 goto init_err;
12651         }
12652
12653         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
12654                 dev_err(&pdev->dev,
12655                         "Cannot find PCI device base address, aborting\n");
12656                 rc = -ENODEV;
12657                 goto init_err_disable;
12658         }
12659
12660         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
12661         if (rc) {
12662                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
12663                 goto init_err_disable;
12664         }
12665
12666         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
12667             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
12668                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
12669                 rc = -EIO;
12670                 goto init_err_release;
12671         }
12672
12673         pci_set_master(pdev);
12674
12675         bp->dev = dev;
12676         bp->pdev = pdev;
12677
12678         /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
12679          * determines the BAR size.
12680          */
12681         bp->bar0 = pci_ioremap_bar(pdev, 0);
12682         if (!bp->bar0) {
12683                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
12684                 rc = -ENOMEM;
12685                 goto init_err_release;
12686         }
12687
12688         bp->bar2 = pci_ioremap_bar(pdev, 4);
12689         if (!bp->bar2) {
12690                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
12691                 rc = -ENOMEM;
12692                 goto init_err_release;
12693         }
12694
12695         INIT_WORK(&bp->sp_task, bnxt_sp_task);
12696         INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
12697
12698         spin_lock_init(&bp->ntp_fltr_lock);
12699 #if BITS_PER_LONG == 32
12700         spin_lock_init(&bp->db_lock);
12701 #endif
12702
12703         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
12704         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
12705
12706         timer_setup(&bp->timer, bnxt_timer, 0);
12707         bp->current_interval = BNXT_TIMER_INTERVAL;
12708
12709         bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
12710         bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
12711
12712         clear_bit(BNXT_STATE_OPEN, &bp->state);
12713         return 0;
12714
12715 init_err_release:
12716         bnxt_unmap_bars(bp, pdev);
12717         pci_release_regions(pdev);
12718
12719 init_err_disable:
12720         pci_disable_device(pdev);
12721
12722 init_err:
12723         return rc;
12724 }
12725
12726 /* rtnl_lock held */
12727 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
12728 {
12729         struct sockaddr *addr = p;
12730         struct bnxt *bp = netdev_priv(dev);
12731         int rc = 0;
12732
12733         if (!is_valid_ether_addr(addr->sa_data))
12734                 return -EADDRNOTAVAIL;
12735
12736         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
12737                 return 0;
12738
12739         rc = bnxt_approve_mac(bp, addr->sa_data, true);
12740         if (rc)
12741                 return rc;
12742
12743         eth_hw_addr_set(dev, addr->sa_data);
12744         if (netif_running(dev)) {
12745                 bnxt_close_nic(bp, false, false);
12746                 rc = bnxt_open_nic(bp, false, false);
12747         }
12748
12749         return rc;
12750 }
12751
12752 /* rtnl_lock held */
12753 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
12754 {
12755         struct bnxt *bp = netdev_priv(dev);
12756
12757         if (netif_running(dev))
12758                 bnxt_close_nic(bp, true, false);
12759
12760         dev->mtu = new_mtu;
12761         bnxt_set_ring_params(bp);
12762
12763         if (netif_running(dev))
12764                 return bnxt_open_nic(bp, true, false);
12765
12766         return 0;
12767 }
12768
12769 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
12770 {
12771         struct bnxt *bp = netdev_priv(dev);
12772         bool sh = false;
12773         int rc;
12774
12775         if (tc > bp->max_tc) {
12776                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
12777                            tc, bp->max_tc);
12778                 return -EINVAL;
12779         }
12780
12781         if (netdev_get_num_tc(dev) == tc)
12782                 return 0;
12783
12784         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
12785                 sh = true;
12786
12787         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
12788                               sh, tc, bp->tx_nr_rings_xdp);
12789         if (rc)
12790                 return rc;
12791
12792         /* Needs to close the device and do hw resource re-allocations */
12793         if (netif_running(bp->dev))
12794                 bnxt_close_nic(bp, true, false);
12795
12796         if (tc) {
12797                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
12798                 netdev_set_num_tc(dev, tc);
12799         } else {
12800                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12801                 netdev_reset_tc(dev);
12802         }
12803         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
12804         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
12805                                bp->tx_nr_rings + bp->rx_nr_rings;
12806
12807         if (netif_running(bp->dev))
12808                 return bnxt_open_nic(bp, true, false);
12809
12810         return 0;
12811 }
12812
12813 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
12814                                   void *cb_priv)
12815 {
12816         struct bnxt *bp = cb_priv;
12817
12818         if (!bnxt_tc_flower_enabled(bp) ||
12819             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
12820                 return -EOPNOTSUPP;
12821
12822         switch (type) {
12823         case TC_SETUP_CLSFLOWER:
12824                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
12825         default:
12826                 return -EOPNOTSUPP;
12827         }
12828 }
12829
12830 LIST_HEAD(bnxt_block_cb_list);
12831
12832 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
12833                          void *type_data)
12834 {
12835         struct bnxt *bp = netdev_priv(dev);
12836
12837         switch (type) {
12838         case TC_SETUP_BLOCK:
12839                 return flow_block_cb_setup_simple(type_data,
12840                                                   &bnxt_block_cb_list,
12841                                                   bnxt_setup_tc_block_cb,
12842                                                   bp, bp, true);
12843         case TC_SETUP_QDISC_MQPRIO: {
12844                 struct tc_mqprio_qopt *mqprio = type_data;
12845
12846                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
12847
12848                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
12849         }
12850         default:
12851                 return -EOPNOTSUPP;
12852         }
12853 }
12854
12855 #ifdef CONFIG_RFS_ACCEL
12856 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
12857                             struct bnxt_ntuple_filter *f2)
12858 {
12859         struct flow_keys *keys1 = &f1->fkeys;
12860         struct flow_keys *keys2 = &f2->fkeys;
12861
12862         if (keys1->basic.n_proto != keys2->basic.n_proto ||
12863             keys1->basic.ip_proto != keys2->basic.ip_proto)
12864                 return false;
12865
12866         if (keys1->basic.n_proto == htons(ETH_P_IP)) {
12867                 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
12868                     keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
12869                         return false;
12870         } else {
12871                 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
12872                            sizeof(keys1->addrs.v6addrs.src)) ||
12873                     memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
12874                            sizeof(keys1->addrs.v6addrs.dst)))
12875                         return false;
12876         }
12877
12878         if (keys1->ports.ports == keys2->ports.ports &&
12879             keys1->control.flags == keys2->control.flags &&
12880             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
12881             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
12882                 return true;
12883
12884         return false;
12885 }
12886
12887 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
12888                               u16 rxq_index, u32 flow_id)
12889 {
12890         struct bnxt *bp = netdev_priv(dev);
12891         struct bnxt_ntuple_filter *fltr, *new_fltr;
12892         struct flow_keys *fkeys;
12893         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
12894         int rc = 0, idx, bit_id, l2_idx = 0;
12895         struct hlist_head *head;
12896         u32 flags;
12897
12898         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
12899                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
12900                 int off = 0, j;
12901
12902                 netif_addr_lock_bh(dev);
12903                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
12904                         if (ether_addr_equal(eth->h_dest,
12905                                              vnic->uc_list + off)) {
12906                                 l2_idx = j + 1;
12907                                 break;
12908                         }
12909                 }
12910                 netif_addr_unlock_bh(dev);
12911                 if (!l2_idx)
12912                         return -EINVAL;
12913         }
12914         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
12915         if (!new_fltr)
12916                 return -ENOMEM;
12917
12918         fkeys = &new_fltr->fkeys;
12919         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
12920                 rc = -EPROTONOSUPPORT;
12921                 goto err_free;
12922         }
12923
12924         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
12925              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
12926             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
12927              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
12928                 rc = -EPROTONOSUPPORT;
12929                 goto err_free;
12930         }
12931         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
12932             bp->hwrm_spec_code < 0x10601) {
12933                 rc = -EPROTONOSUPPORT;
12934                 goto err_free;
12935         }
12936         flags = fkeys->control.flags;
12937         if (((flags & FLOW_DIS_ENCAPSULATION) &&
12938              bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
12939                 rc = -EPROTONOSUPPORT;
12940                 goto err_free;
12941         }
12942
12943         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
12944         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
12945
12946         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
12947         head = &bp->ntp_fltr_hash_tbl[idx];
12948         rcu_read_lock();
12949         hlist_for_each_entry_rcu(fltr, head, hash) {
12950                 if (bnxt_fltr_match(fltr, new_fltr)) {
12951                         rc = fltr->sw_id;
12952                         rcu_read_unlock();
12953                         goto err_free;
12954                 }
12955         }
12956         rcu_read_unlock();
12957
12958         spin_lock_bh(&bp->ntp_fltr_lock);
12959         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
12960                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
12961         if (bit_id < 0) {
12962                 spin_unlock_bh(&bp->ntp_fltr_lock);
12963                 rc = -ENOMEM;
12964                 goto err_free;
12965         }
12966
12967         new_fltr->sw_id = (u16)bit_id;
12968         new_fltr->flow_id = flow_id;
12969         new_fltr->l2_fltr_idx = l2_idx;
12970         new_fltr->rxq = rxq_index;
12971         hlist_add_head_rcu(&new_fltr->hash, head);
12972         bp->ntp_fltr_count++;
12973         spin_unlock_bh(&bp->ntp_fltr_lock);
12974
12975         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
12976         bnxt_queue_sp_work(bp);
12977
12978         return new_fltr->sw_id;
12979
12980 err_free:
12981         kfree(new_fltr);
12982         return rc;
12983 }
12984
12985 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
12986 {
12987         int i;
12988
12989         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
12990                 struct hlist_head *head;
12991                 struct hlist_node *tmp;
12992                 struct bnxt_ntuple_filter *fltr;
12993                 int rc;
12994
12995                 head = &bp->ntp_fltr_hash_tbl[i];
12996                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
12997                         bool del = false;
12998
12999                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
13000                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
13001                                                         fltr->flow_id,
13002                                                         fltr->sw_id)) {
13003                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
13004                                                                          fltr);
13005                                         del = true;
13006                                 }
13007                         } else {
13008                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
13009                                                                        fltr);
13010                                 if (rc)
13011                                         del = true;
13012                                 else
13013                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
13014                         }
13015
13016                         if (del) {
13017                                 spin_lock_bh(&bp->ntp_fltr_lock);
13018                                 hlist_del_rcu(&fltr->hash);
13019                                 bp->ntp_fltr_count--;
13020                                 spin_unlock_bh(&bp->ntp_fltr_lock);
13021                                 synchronize_rcu();
13022                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
13023                                 kfree(fltr);
13024                         }
13025                 }
13026         }
13027         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
13028                 netdev_info(bp->dev, "Receive PF driver unload event!\n");
13029 }
13030
13031 #else
13032
13033 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
13034 {
13035 }
13036
13037 #endif /* CONFIG_RFS_ACCEL */
13038
13039 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table)
13040 {
13041         struct bnxt *bp = netdev_priv(netdev);
13042         struct udp_tunnel_info ti;
13043         unsigned int cmd;
13044
13045         udp_tunnel_nic_get_port(netdev, table, 0, &ti);
13046         if (ti.type == UDP_TUNNEL_TYPE_VXLAN)
13047                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
13048         else
13049                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
13050
13051         if (ti.port)
13052                 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd);
13053
13054         return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
13055 }
13056
13057 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
13058         .sync_table     = bnxt_udp_tunnel_sync,
13059         .flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
13060                           UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
13061         .tables         = {
13062                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
13063                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
13064         },
13065 };
13066
13067 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
13068                                struct net_device *dev, u32 filter_mask,
13069                                int nlflags)
13070 {
13071         struct bnxt *bp = netdev_priv(dev);
13072
13073         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
13074                                        nlflags, filter_mask, NULL);
13075 }
13076
13077 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
13078                                u16 flags, struct netlink_ext_ack *extack)
13079 {
13080         struct bnxt *bp = netdev_priv(dev);
13081         struct nlattr *attr, *br_spec;
13082         int rem, rc = 0;
13083
13084         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
13085                 return -EOPNOTSUPP;
13086
13087         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
13088         if (!br_spec)
13089                 return -EINVAL;
13090
13091         nla_for_each_nested(attr, br_spec, rem) {
13092                 u16 mode;
13093
13094                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
13095                         continue;
13096
13097                 if (nla_len(attr) < sizeof(mode))
13098                         return -EINVAL;
13099
13100                 mode = nla_get_u16(attr);
13101                 if (mode == bp->br_mode)
13102                         break;
13103
13104                 rc = bnxt_hwrm_set_br_mode(bp, mode);
13105                 if (!rc)
13106                         bp->br_mode = mode;
13107                 break;
13108         }
13109         return rc;
13110 }
13111
13112 int bnxt_get_port_parent_id(struct net_device *dev,
13113                             struct netdev_phys_item_id *ppid)
13114 {
13115         struct bnxt *bp = netdev_priv(dev);
13116
13117         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
13118                 return -EOPNOTSUPP;
13119
13120         /* The PF and it's VF-reps only support the switchdev framework */
13121         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
13122                 return -EOPNOTSUPP;
13123
13124         ppid->id_len = sizeof(bp->dsn);
13125         memcpy(ppid->id, bp->dsn, ppid->id_len);
13126
13127         return 0;
13128 }
13129
13130 static const struct net_device_ops bnxt_netdev_ops = {
13131         .ndo_open               = bnxt_open,
13132         .ndo_start_xmit         = bnxt_start_xmit,
13133         .ndo_stop               = bnxt_close,
13134         .ndo_get_stats64        = bnxt_get_stats64,
13135         .ndo_set_rx_mode        = bnxt_set_rx_mode,
13136         .ndo_eth_ioctl          = bnxt_ioctl,
13137         .ndo_validate_addr      = eth_validate_addr,
13138         .ndo_set_mac_address    = bnxt_change_mac_addr,
13139         .ndo_change_mtu         = bnxt_change_mtu,
13140         .ndo_fix_features       = bnxt_fix_features,
13141         .ndo_set_features       = bnxt_set_features,
13142         .ndo_features_check     = bnxt_features_check,
13143         .ndo_tx_timeout         = bnxt_tx_timeout,
13144 #ifdef CONFIG_BNXT_SRIOV
13145         .ndo_get_vf_config      = bnxt_get_vf_config,
13146         .ndo_set_vf_mac         = bnxt_set_vf_mac,
13147         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
13148         .ndo_set_vf_rate        = bnxt_set_vf_bw,
13149         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
13150         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
13151         .ndo_set_vf_trust       = bnxt_set_vf_trust,
13152 #endif
13153         .ndo_setup_tc           = bnxt_setup_tc,
13154 #ifdef CONFIG_RFS_ACCEL
13155         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
13156 #endif
13157         .ndo_bpf                = bnxt_xdp,
13158         .ndo_xdp_xmit           = bnxt_xdp_xmit,
13159         .ndo_bridge_getlink     = bnxt_bridge_getlink,
13160         .ndo_bridge_setlink     = bnxt_bridge_setlink,
13161 };
13162
13163 static void bnxt_remove_one(struct pci_dev *pdev)
13164 {
13165         struct net_device *dev = pci_get_drvdata(pdev);
13166         struct bnxt *bp = netdev_priv(dev);
13167
13168         if (BNXT_PF(bp))
13169                 bnxt_sriov_disable(bp);
13170
13171         bnxt_rdma_aux_device_uninit(bp);
13172
13173         bnxt_ptp_clear(bp);
13174         unregister_netdev(dev);
13175         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13176         /* Flush any pending tasks */
13177         cancel_work_sync(&bp->sp_task);
13178         cancel_delayed_work_sync(&bp->fw_reset_task);
13179         bp->sp_event = 0;
13180
13181         bnxt_dl_fw_reporters_destroy(bp);
13182         bnxt_dl_unregister(bp);
13183         bnxt_shutdown_tc(bp);
13184
13185         bnxt_clear_int_mode(bp);
13186         bnxt_hwrm_func_drv_unrgtr(bp);
13187         bnxt_free_hwrm_resources(bp);
13188         bnxt_ethtool_free(bp);
13189         bnxt_dcb_free(bp);
13190         kfree(bp->ptp_cfg);
13191         bp->ptp_cfg = NULL;
13192         kfree(bp->fw_health);
13193         bp->fw_health = NULL;
13194         bnxt_cleanup_pci(bp);
13195         bnxt_free_ctx_mem(bp);
13196         kfree(bp->ctx);
13197         bp->ctx = NULL;
13198         kfree(bp->rss_indir_tbl);
13199         bp->rss_indir_tbl = NULL;
13200         bnxt_free_port_stats(bp);
13201         free_netdev(dev);
13202 }
13203
13204 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
13205 {
13206         int rc = 0;
13207         struct bnxt_link_info *link_info = &bp->link_info;
13208
13209         bp->phy_flags = 0;
13210         rc = bnxt_hwrm_phy_qcaps(bp);
13211         if (rc) {
13212                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
13213                            rc);
13214                 return rc;
13215         }
13216         if (bp->phy_flags & BNXT_PHY_FL_NO_FCS)
13217                 bp->dev->priv_flags |= IFF_SUPP_NOFCS;
13218         else
13219                 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS;
13220         if (!fw_dflt)
13221                 return 0;
13222
13223         mutex_lock(&bp->link_lock);
13224         rc = bnxt_update_link(bp, false);
13225         if (rc) {
13226                 mutex_unlock(&bp->link_lock);
13227                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
13228                            rc);
13229                 return rc;
13230         }
13231
13232         /* Older firmware does not have supported_auto_speeds, so assume
13233          * that all supported speeds can be autonegotiated.
13234          */
13235         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
13236                 link_info->support_auto_speeds = link_info->support_speeds;
13237
13238         bnxt_init_ethtool_link_settings(bp);
13239         mutex_unlock(&bp->link_lock);
13240         return 0;
13241 }
13242
13243 static int bnxt_get_max_irq(struct pci_dev *pdev)
13244 {
13245         u16 ctrl;
13246
13247         if (!pdev->msix_cap)
13248                 return 1;
13249
13250         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
13251         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
13252 }
13253
13254 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13255                                 int *max_cp)
13256 {
13257         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
13258         int max_ring_grps = 0, max_irq;
13259
13260         *max_tx = hw_resc->max_tx_rings;
13261         *max_rx = hw_resc->max_rx_rings;
13262         *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
13263         max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
13264                         bnxt_get_ulp_msix_num(bp),
13265                         hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
13266         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
13267                 *max_cp = min_t(int, *max_cp, max_irq);
13268         max_ring_grps = hw_resc->max_hw_ring_grps;
13269         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
13270                 *max_cp -= 1;
13271                 *max_rx -= 2;
13272         }
13273         if (bp->flags & BNXT_FLAG_AGG_RINGS)
13274                 *max_rx >>= 1;
13275         if (bp->flags & BNXT_FLAG_CHIP_P5) {
13276                 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
13277                 /* On P5 chips, max_cp output param should be available NQs */
13278                 *max_cp = max_irq;
13279         }
13280         *max_rx = min_t(int, *max_rx, max_ring_grps);
13281 }
13282
13283 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
13284 {
13285         int rx, tx, cp;
13286
13287         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
13288         *max_rx = rx;
13289         *max_tx = tx;
13290         if (!rx || !tx || !cp)
13291                 return -ENOMEM;
13292
13293         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
13294 }
13295
13296 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13297                                bool shared)
13298 {
13299         int rc;
13300
13301         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
13302         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
13303                 /* Not enough rings, try disabling agg rings. */
13304                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
13305                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
13306                 if (rc) {
13307                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
13308                         bp->flags |= BNXT_FLAG_AGG_RINGS;
13309                         return rc;
13310                 }
13311                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
13312                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
13313                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
13314                 bnxt_set_ring_params(bp);
13315         }
13316
13317         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
13318                 int max_cp, max_stat, max_irq;
13319
13320                 /* Reserve minimum resources for RoCE */
13321                 max_cp = bnxt_get_max_func_cp_rings(bp);
13322                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
13323                 max_irq = bnxt_get_max_func_irqs(bp);
13324                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
13325                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
13326                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
13327                         return 0;
13328
13329                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
13330                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
13331                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
13332                 max_cp = min_t(int, max_cp, max_irq);
13333                 max_cp = min_t(int, max_cp, max_stat);
13334                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
13335                 if (rc)
13336                         rc = 0;
13337         }
13338         return rc;
13339 }
13340
13341 /* In initial default shared ring setting, each shared ring must have a
13342  * RX/TX ring pair.
13343  */
13344 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
13345 {
13346         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
13347         bp->rx_nr_rings = bp->cp_nr_rings;
13348         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
13349         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
13350 }
13351
13352 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
13353 {
13354         int dflt_rings, max_rx_rings, max_tx_rings, rc;
13355
13356         if (!bnxt_can_reserve_rings(bp))
13357                 return 0;
13358
13359         if (sh)
13360                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
13361         dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
13362         /* Reduce default rings on multi-port cards so that total default
13363          * rings do not exceed CPU count.
13364          */
13365         if (bp->port_count > 1) {
13366                 int max_rings =
13367                         max_t(int, num_online_cpus() / bp->port_count, 1);
13368
13369                 dflt_rings = min_t(int, dflt_rings, max_rings);
13370         }
13371         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
13372         if (rc)
13373                 return rc;
13374         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
13375         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
13376         if (sh)
13377                 bnxt_trim_dflt_sh_rings(bp);
13378         else
13379                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
13380         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
13381
13382         rc = __bnxt_reserve_rings(bp);
13383         if (rc && rc != -ENODEV)
13384                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
13385         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13386         if (sh)
13387                 bnxt_trim_dflt_sh_rings(bp);
13388
13389         /* Rings may have been trimmed, re-reserve the trimmed rings. */
13390         if (bnxt_need_reserve_rings(bp)) {
13391                 rc = __bnxt_reserve_rings(bp);
13392                 if (rc && rc != -ENODEV)
13393                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
13394                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13395         }
13396         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
13397                 bp->rx_nr_rings++;
13398                 bp->cp_nr_rings++;
13399         }
13400         if (rc) {
13401                 bp->tx_nr_rings = 0;
13402                 bp->rx_nr_rings = 0;
13403         }
13404         return rc;
13405 }
13406
13407 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
13408 {
13409         int rc;
13410
13411         if (bp->tx_nr_rings)
13412                 return 0;
13413
13414         bnxt_ulp_irq_stop(bp);
13415         bnxt_clear_int_mode(bp);
13416         rc = bnxt_set_dflt_rings(bp, true);
13417         if (rc) {
13418                 if (BNXT_VF(bp) && rc == -ENODEV)
13419                         netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
13420                 else
13421                         netdev_err(bp->dev, "Not enough rings available.\n");
13422                 goto init_dflt_ring_err;
13423         }
13424         rc = bnxt_init_int_mode(bp);
13425         if (rc)
13426                 goto init_dflt_ring_err;
13427
13428         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13429
13430         bnxt_set_dflt_rfs(bp);
13431
13432 init_dflt_ring_err:
13433         bnxt_ulp_irq_restart(bp, rc);
13434         return rc;
13435 }
13436
13437 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
13438 {
13439         int rc;
13440
13441         ASSERT_RTNL();
13442         bnxt_hwrm_func_qcaps(bp);
13443
13444         if (netif_running(bp->dev))
13445                 __bnxt_close_nic(bp, true, false);
13446
13447         bnxt_ulp_irq_stop(bp);
13448         bnxt_clear_int_mode(bp);
13449         rc = bnxt_init_int_mode(bp);
13450         bnxt_ulp_irq_restart(bp, rc);
13451
13452         if (netif_running(bp->dev)) {
13453                 if (rc)
13454                         dev_close(bp->dev);
13455                 else
13456                         rc = bnxt_open_nic(bp, true, false);
13457         }
13458
13459         return rc;
13460 }
13461
13462 static int bnxt_init_mac_addr(struct bnxt *bp)
13463 {
13464         int rc = 0;
13465
13466         if (BNXT_PF(bp)) {
13467                 eth_hw_addr_set(bp->dev, bp->pf.mac_addr);
13468         } else {
13469 #ifdef CONFIG_BNXT_SRIOV
13470                 struct bnxt_vf_info *vf = &bp->vf;
13471                 bool strict_approval = true;
13472
13473                 if (is_valid_ether_addr(vf->mac_addr)) {
13474                         /* overwrite netdev dev_addr with admin VF MAC */
13475                         eth_hw_addr_set(bp->dev, vf->mac_addr);
13476                         /* Older PF driver or firmware may not approve this
13477                          * correctly.
13478                          */
13479                         strict_approval = false;
13480                 } else {
13481                         eth_hw_addr_random(bp->dev);
13482                 }
13483                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
13484 #endif
13485         }
13486         return rc;
13487 }
13488
13489 static void bnxt_vpd_read_info(struct bnxt *bp)
13490 {
13491         struct pci_dev *pdev = bp->pdev;
13492         unsigned int vpd_size, kw_len;
13493         int pos, size;
13494         u8 *vpd_data;
13495
13496         vpd_data = pci_vpd_alloc(pdev, &vpd_size);
13497         if (IS_ERR(vpd_data)) {
13498                 pci_warn(pdev, "Unable to read VPD\n");
13499                 return;
13500         }
13501
13502         pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
13503                                            PCI_VPD_RO_KEYWORD_PARTNO, &kw_len);
13504         if (pos < 0)
13505                 goto read_sn;
13506
13507         size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
13508         memcpy(bp->board_partno, &vpd_data[pos], size);
13509
13510 read_sn:
13511         pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
13512                                            PCI_VPD_RO_KEYWORD_SERIALNO,
13513                                            &kw_len);
13514         if (pos < 0)
13515                 goto exit;
13516
13517         size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
13518         memcpy(bp->board_serialno, &vpd_data[pos], size);
13519 exit:
13520         kfree(vpd_data);
13521 }
13522
13523 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
13524 {
13525         struct pci_dev *pdev = bp->pdev;
13526         u64 qword;
13527
13528         qword = pci_get_dsn(pdev);
13529         if (!qword) {
13530                 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
13531                 return -EOPNOTSUPP;
13532         }
13533
13534         put_unaligned_le64(qword, dsn);
13535
13536         bp->flags |= BNXT_FLAG_DSN_VALID;
13537         return 0;
13538 }
13539
13540 static int bnxt_map_db_bar(struct bnxt *bp)
13541 {
13542         if (!bp->db_size)
13543                 return -ENODEV;
13544         bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
13545         if (!bp->bar1)
13546                 return -ENOMEM;
13547         return 0;
13548 }
13549
13550 void bnxt_print_device_info(struct bnxt *bp)
13551 {
13552         netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n",
13553                     board_info[bp->board_idx].name,
13554                     (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr);
13555
13556         pcie_print_link_status(bp->pdev);
13557 }
13558
13559 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
13560 {
13561         struct net_device *dev;
13562         struct bnxt *bp;
13563         int rc, max_irqs;
13564
13565         if (pci_is_bridge(pdev))
13566                 return -ENODEV;
13567
13568         /* Clear any pending DMA transactions from crash kernel
13569          * while loading driver in capture kernel.
13570          */
13571         if (is_kdump_kernel()) {
13572                 pci_clear_master(pdev);
13573                 pcie_flr(pdev);
13574         }
13575
13576         max_irqs = bnxt_get_max_irq(pdev);
13577         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
13578         if (!dev)
13579                 return -ENOMEM;
13580
13581         bp = netdev_priv(dev);
13582         bp->board_idx = ent->driver_data;
13583         bp->msg_enable = BNXT_DEF_MSG_ENABLE;
13584         bnxt_set_max_func_irqs(bp, max_irqs);
13585
13586         if (bnxt_vf_pciid(bp->board_idx))
13587                 bp->flags |= BNXT_FLAG_VF;
13588
13589         /* No devlink port registration in case of a VF */
13590         if (BNXT_PF(bp))
13591                 SET_NETDEV_DEVLINK_PORT(dev, &bp->dl_port);
13592
13593         if (pdev->msix_cap)
13594                 bp->flags |= BNXT_FLAG_MSIX_CAP;
13595
13596         rc = bnxt_init_board(pdev, dev);
13597         if (rc < 0)
13598                 goto init_err_free;
13599
13600         dev->netdev_ops = &bnxt_netdev_ops;
13601         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
13602         dev->ethtool_ops = &bnxt_ethtool_ops;
13603         pci_set_drvdata(pdev, dev);
13604
13605         rc = bnxt_alloc_hwrm_resources(bp);
13606         if (rc)
13607                 goto init_err_pci_clean;
13608
13609         mutex_init(&bp->hwrm_cmd_lock);
13610         mutex_init(&bp->link_lock);
13611
13612         rc = bnxt_fw_init_one_p1(bp);
13613         if (rc)
13614                 goto init_err_pci_clean;
13615
13616         if (BNXT_PF(bp))
13617                 bnxt_vpd_read_info(bp);
13618
13619         if (BNXT_CHIP_P5(bp)) {
13620                 bp->flags |= BNXT_FLAG_CHIP_P5;
13621                 if (BNXT_CHIP_SR2(bp))
13622                         bp->flags |= BNXT_FLAG_CHIP_SR2;
13623         }
13624
13625         rc = bnxt_alloc_rss_indir_tbl(bp);
13626         if (rc)
13627                 goto init_err_pci_clean;
13628
13629         rc = bnxt_fw_init_one_p2(bp);
13630         if (rc)
13631                 goto init_err_pci_clean;
13632
13633         rc = bnxt_map_db_bar(bp);
13634         if (rc) {
13635                 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
13636                         rc);
13637                 goto init_err_pci_clean;
13638         }
13639
13640         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
13641                            NETIF_F_TSO | NETIF_F_TSO6 |
13642                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
13643                            NETIF_F_GSO_IPXIP4 |
13644                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
13645                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
13646                            NETIF_F_RXCSUM | NETIF_F_GRO;
13647
13648         if (BNXT_SUPPORTS_TPA(bp))
13649                 dev->hw_features |= NETIF_F_LRO;
13650
13651         dev->hw_enc_features =
13652                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
13653                         NETIF_F_TSO | NETIF_F_TSO6 |
13654                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
13655                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
13656                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
13657         dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
13658
13659         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
13660                                     NETIF_F_GSO_GRE_CSUM;
13661         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
13662         if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
13663                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
13664         if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
13665                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
13666         if (BNXT_SUPPORTS_TPA(bp))
13667                 dev->hw_features |= NETIF_F_GRO_HW;
13668         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
13669         if (dev->features & NETIF_F_GRO_HW)
13670                 dev->features &= ~NETIF_F_LRO;
13671         dev->priv_flags |= IFF_UNICAST_FLT;
13672
13673         netif_set_tso_max_size(dev, GSO_MAX_SIZE);
13674
13675         dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
13676                             NETDEV_XDP_ACT_RX_SG;
13677
13678 #ifdef CONFIG_BNXT_SRIOV
13679         init_waitqueue_head(&bp->sriov_cfg_wait);
13680 #endif
13681         if (BNXT_SUPPORTS_TPA(bp)) {
13682                 bp->gro_func = bnxt_gro_func_5730x;
13683                 if (BNXT_CHIP_P4(bp))
13684                         bp->gro_func = bnxt_gro_func_5731x;
13685                 else if (BNXT_CHIP_P5(bp))
13686                         bp->gro_func = bnxt_gro_func_5750x;
13687         }
13688         if (!BNXT_CHIP_P4_PLUS(bp))
13689                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
13690
13691         rc = bnxt_init_mac_addr(bp);
13692         if (rc) {
13693                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
13694                 rc = -EADDRNOTAVAIL;
13695                 goto init_err_pci_clean;
13696         }
13697
13698         if (BNXT_PF(bp)) {
13699                 /* Read the adapter's DSN to use as the eswitch switch_id */
13700                 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
13701         }
13702
13703         /* MTU range: 60 - FW defined max */
13704         dev->min_mtu = ETH_ZLEN;
13705         dev->max_mtu = bp->max_mtu;
13706
13707         rc = bnxt_probe_phy(bp, true);
13708         if (rc)
13709                 goto init_err_pci_clean;
13710
13711         bnxt_set_rx_skb_mode(bp, false);
13712         bnxt_set_tpa_flags(bp);
13713         bnxt_set_ring_params(bp);
13714         rc = bnxt_set_dflt_rings(bp, true);
13715         if (rc) {
13716                 if (BNXT_VF(bp) && rc == -ENODEV) {
13717                         netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
13718                 } else {
13719                         netdev_err(bp->dev, "Not enough rings available.\n");
13720                         rc = -ENOMEM;
13721                 }
13722                 goto init_err_pci_clean;
13723         }
13724
13725         bnxt_fw_init_one_p3(bp);
13726
13727         bnxt_init_dflt_coal(bp);
13728
13729         if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
13730                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
13731
13732         rc = bnxt_init_int_mode(bp);
13733         if (rc)
13734                 goto init_err_pci_clean;
13735
13736         /* No TC has been set yet and rings may have been trimmed due to
13737          * limited MSIX, so we re-initialize the TX rings per TC.
13738          */
13739         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13740
13741         if (BNXT_PF(bp)) {
13742                 if (!bnxt_pf_wq) {
13743                         bnxt_pf_wq =
13744                                 create_singlethread_workqueue("bnxt_pf_wq");
13745                         if (!bnxt_pf_wq) {
13746                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
13747                                 rc = -ENOMEM;
13748                                 goto init_err_pci_clean;
13749                         }
13750                 }
13751                 rc = bnxt_init_tc(bp);
13752                 if (rc)
13753                         netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
13754                                    rc);
13755         }
13756
13757         bnxt_inv_fw_health_reg(bp);
13758         rc = bnxt_dl_register(bp);
13759         if (rc)
13760                 goto init_err_dl;
13761
13762         rc = register_netdev(dev);
13763         if (rc)
13764                 goto init_err_cleanup;
13765
13766         bnxt_dl_fw_reporters_create(bp);
13767
13768         bnxt_rdma_aux_device_init(bp);
13769
13770         bnxt_print_device_info(bp);
13771
13772         pci_save_state(pdev);
13773
13774         return 0;
13775 init_err_cleanup:
13776         bnxt_dl_unregister(bp);
13777 init_err_dl:
13778         bnxt_shutdown_tc(bp);
13779         bnxt_clear_int_mode(bp);
13780
13781 init_err_pci_clean:
13782         bnxt_hwrm_func_drv_unrgtr(bp);
13783         bnxt_free_hwrm_resources(bp);
13784         bnxt_ethtool_free(bp);
13785         bnxt_ptp_clear(bp);
13786         kfree(bp->ptp_cfg);
13787         bp->ptp_cfg = NULL;
13788         kfree(bp->fw_health);
13789         bp->fw_health = NULL;
13790         bnxt_cleanup_pci(bp);
13791         bnxt_free_ctx_mem(bp);
13792         kfree(bp->ctx);
13793         bp->ctx = NULL;
13794         kfree(bp->rss_indir_tbl);
13795         bp->rss_indir_tbl = NULL;
13796
13797 init_err_free:
13798         free_netdev(dev);
13799         return rc;
13800 }
13801
13802 static void bnxt_shutdown(struct pci_dev *pdev)
13803 {
13804         struct net_device *dev = pci_get_drvdata(pdev);
13805         struct bnxt *bp;
13806
13807         if (!dev)
13808                 return;
13809
13810         rtnl_lock();
13811         bp = netdev_priv(dev);
13812         if (!bp)
13813                 goto shutdown_exit;
13814
13815         if (netif_running(dev))
13816                 dev_close(dev);
13817
13818         bnxt_clear_int_mode(bp);
13819         pci_disable_device(pdev);
13820
13821         if (system_state == SYSTEM_POWER_OFF) {
13822                 pci_wake_from_d3(pdev, bp->wol);
13823                 pci_set_power_state(pdev, PCI_D3hot);
13824         }
13825
13826 shutdown_exit:
13827         rtnl_unlock();
13828 }
13829
13830 #ifdef CONFIG_PM_SLEEP
13831 static int bnxt_suspend(struct device *device)
13832 {
13833         struct net_device *dev = dev_get_drvdata(device);
13834         struct bnxt *bp = netdev_priv(dev);
13835         int rc = 0;
13836
13837         rtnl_lock();
13838         bnxt_ulp_stop(bp);
13839         if (netif_running(dev)) {
13840                 netif_device_detach(dev);
13841                 rc = bnxt_close(dev);
13842         }
13843         bnxt_hwrm_func_drv_unrgtr(bp);
13844         pci_disable_device(bp->pdev);
13845         bnxt_free_ctx_mem(bp);
13846         kfree(bp->ctx);
13847         bp->ctx = NULL;
13848         rtnl_unlock();
13849         return rc;
13850 }
13851
13852 static int bnxt_resume(struct device *device)
13853 {
13854         struct net_device *dev = dev_get_drvdata(device);
13855         struct bnxt *bp = netdev_priv(dev);
13856         int rc = 0;
13857
13858         rtnl_lock();
13859         rc = pci_enable_device(bp->pdev);
13860         if (rc) {
13861                 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
13862                            rc);
13863                 goto resume_exit;
13864         }
13865         pci_set_master(bp->pdev);
13866         if (bnxt_hwrm_ver_get(bp)) {
13867                 rc = -ENODEV;
13868                 goto resume_exit;
13869         }
13870         rc = bnxt_hwrm_func_reset(bp);
13871         if (rc) {
13872                 rc = -EBUSY;
13873                 goto resume_exit;
13874         }
13875
13876         rc = bnxt_hwrm_func_qcaps(bp);
13877         if (rc)
13878                 goto resume_exit;
13879
13880         if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
13881                 rc = -ENODEV;
13882                 goto resume_exit;
13883         }
13884
13885         bnxt_get_wol_settings(bp);
13886         if (netif_running(dev)) {
13887                 rc = bnxt_open(dev);
13888                 if (!rc)
13889                         netif_device_attach(dev);
13890         }
13891
13892 resume_exit:
13893         bnxt_ulp_start(bp, rc);
13894         if (!rc)
13895                 bnxt_reenable_sriov(bp);
13896         rtnl_unlock();
13897         return rc;
13898 }
13899
13900 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
13901 #define BNXT_PM_OPS (&bnxt_pm_ops)
13902
13903 #else
13904
13905 #define BNXT_PM_OPS NULL
13906
13907 #endif /* CONFIG_PM_SLEEP */
13908
13909 /**
13910  * bnxt_io_error_detected - called when PCI error is detected
13911  * @pdev: Pointer to PCI device
13912  * @state: The current pci connection state
13913  *
13914  * This function is called after a PCI bus error affecting
13915  * this device has been detected.
13916  */
13917 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
13918                                                pci_channel_state_t state)
13919 {
13920         struct net_device *netdev = pci_get_drvdata(pdev);
13921         struct bnxt *bp = netdev_priv(netdev);
13922
13923         netdev_info(netdev, "PCI I/O error detected\n");
13924
13925         rtnl_lock();
13926         netif_device_detach(netdev);
13927
13928         bnxt_ulp_stop(bp);
13929
13930         if (state == pci_channel_io_perm_failure) {
13931                 rtnl_unlock();
13932                 return PCI_ERS_RESULT_DISCONNECT;
13933         }
13934
13935         if (state == pci_channel_io_frozen)
13936                 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
13937
13938         if (netif_running(netdev))
13939                 bnxt_close(netdev);
13940
13941         if (pci_is_enabled(pdev))
13942                 pci_disable_device(pdev);
13943         bnxt_free_ctx_mem(bp);
13944         kfree(bp->ctx);
13945         bp->ctx = NULL;
13946         rtnl_unlock();
13947
13948         /* Request a slot slot reset. */
13949         return PCI_ERS_RESULT_NEED_RESET;
13950 }
13951
13952 /**
13953  * bnxt_io_slot_reset - called after the pci bus has been reset.
13954  * @pdev: Pointer to PCI device
13955  *
13956  * Restart the card from scratch, as if from a cold-boot.
13957  * At this point, the card has exprienced a hard reset,
13958  * followed by fixups by BIOS, and has its config space
13959  * set up identically to what it was at cold boot.
13960  */
13961 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
13962 {
13963         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
13964         struct net_device *netdev = pci_get_drvdata(pdev);
13965         struct bnxt *bp = netdev_priv(netdev);
13966         int retry = 0;
13967         int err = 0;
13968         int off;
13969
13970         netdev_info(bp->dev, "PCI Slot Reset\n");
13971
13972         rtnl_lock();
13973
13974         if (pci_enable_device(pdev)) {
13975                 dev_err(&pdev->dev,
13976                         "Cannot re-enable PCI device after reset.\n");
13977         } else {
13978                 pci_set_master(pdev);
13979                 /* Upon fatal error, our device internal logic that latches to
13980                  * BAR value is getting reset and will restore only upon
13981                  * rewritting the BARs.
13982                  *
13983                  * As pci_restore_state() does not re-write the BARs if the
13984                  * value is same as saved value earlier, driver needs to
13985                  * write the BARs to 0 to force restore, in case of fatal error.
13986                  */
13987                 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
13988                                        &bp->state)) {
13989                         for (off = PCI_BASE_ADDRESS_0;
13990                              off <= PCI_BASE_ADDRESS_5; off += 4)
13991                                 pci_write_config_dword(bp->pdev, off, 0);
13992                 }
13993                 pci_restore_state(pdev);
13994                 pci_save_state(pdev);
13995
13996                 bnxt_inv_fw_health_reg(bp);
13997                 bnxt_try_map_fw_health_reg(bp);
13998
13999                 /* In some PCIe AER scenarios, firmware may take up to
14000                  * 10 seconds to become ready in the worst case.
14001                  */
14002                 do {
14003                         err = bnxt_try_recover_fw(bp);
14004                         if (!err)
14005                                 break;
14006                         retry++;
14007                 } while (retry < BNXT_FW_SLOT_RESET_RETRY);
14008
14009                 if (err) {
14010                         dev_err(&pdev->dev, "Firmware not ready\n");
14011                         goto reset_exit;
14012                 }
14013
14014                 err = bnxt_hwrm_func_reset(bp);
14015                 if (!err)
14016                         result = PCI_ERS_RESULT_RECOVERED;
14017
14018                 bnxt_ulp_irq_stop(bp);
14019                 bnxt_clear_int_mode(bp);
14020                 err = bnxt_init_int_mode(bp);
14021                 bnxt_ulp_irq_restart(bp, err);
14022         }
14023
14024 reset_exit:
14025         bnxt_clear_reservations(bp, true);
14026         rtnl_unlock();
14027
14028         return result;
14029 }
14030
14031 /**
14032  * bnxt_io_resume - called when traffic can start flowing again.
14033  * @pdev: Pointer to PCI device
14034  *
14035  * This callback is called when the error recovery driver tells
14036  * us that its OK to resume normal operation.
14037  */
14038 static void bnxt_io_resume(struct pci_dev *pdev)
14039 {
14040         struct net_device *netdev = pci_get_drvdata(pdev);
14041         struct bnxt *bp = netdev_priv(netdev);
14042         int err;
14043
14044         netdev_info(bp->dev, "PCI Slot Resume\n");
14045         rtnl_lock();
14046
14047         err = bnxt_hwrm_func_qcaps(bp);
14048         if (!err && netif_running(netdev))
14049                 err = bnxt_open(netdev);
14050
14051         bnxt_ulp_start(bp, err);
14052         if (!err) {
14053                 bnxt_reenable_sriov(bp);
14054                 netif_device_attach(netdev);
14055         }
14056
14057         rtnl_unlock();
14058 }
14059
14060 static const struct pci_error_handlers bnxt_err_handler = {
14061         .error_detected = bnxt_io_error_detected,
14062         .slot_reset     = bnxt_io_slot_reset,
14063         .resume         = bnxt_io_resume
14064 };
14065
14066 static struct pci_driver bnxt_pci_driver = {
14067         .name           = DRV_MODULE_NAME,
14068         .id_table       = bnxt_pci_tbl,
14069         .probe          = bnxt_init_one,
14070         .remove         = bnxt_remove_one,
14071         .shutdown       = bnxt_shutdown,
14072         .driver.pm      = BNXT_PM_OPS,
14073         .err_handler    = &bnxt_err_handler,
14074 #if defined(CONFIG_BNXT_SRIOV)
14075         .sriov_configure = bnxt_sriov_configure,
14076 #endif
14077 };
14078
14079 static int __init bnxt_init(void)
14080 {
14081         int err;
14082
14083         bnxt_debug_init();
14084         err = pci_register_driver(&bnxt_pci_driver);
14085         if (err) {
14086                 bnxt_debug_exit();
14087                 return err;
14088         }
14089
14090         return 0;
14091 }
14092
14093 static void __exit bnxt_exit(void)
14094 {
14095         pci_unregister_driver(&bnxt_pci_driver);
14096         if (bnxt_pf_wq)
14097                 destroy_workqueue(bnxt_pf_wq);
14098         bnxt_debug_exit();
14099 }
14100
14101 module_init(bnxt_init);
14102 module_exit(bnxt_exit);