drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2018 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/if.h>
  35 #include <linux/if_vlan.h>
  36 #include <linux/if_bridge.h>
  37 #include <linux/rtc.h>
  38 #include <linux/bpf.h>
  39 #include <net/ip.h>
  40 #include <net/tcp.h>
  41 #include <net/udp.h>
  42 #include <net/checksum.h>
  43 #include <net/ip6_checksum.h>
  44 #include <net/udp_tunnel.h>
  45 #include <linux/workqueue.h>
  46 #include <linux/prefetch.h>
  47 #include <linux/cache.h>
  48 #include <linux/log2.h>
  49 #include <linux/aer.h>
  50 #include <linux/bitmap.h>
  51 #include <linux/cpu_rmap.h>
  52 #include <linux/cpumask.h>
  53 #include <net/pkt_cls.h>
  54 #include <linux/hwmon.h>
  55 #include <linux/hwmon-sysfs.h>
  56
  57 #include "bnxt_hsi.h"
  58 #include "bnxt.h"
  59 #include "bnxt_ulp.h"
  60 #include "bnxt_sriov.h"
  61 #include "bnxt_ethtool.h"
  62 #include "bnxt_dcb.h"
  63 #include "bnxt_xdp.h"
  64 #include "bnxt_vfr.h"
  65 #include "bnxt_tc.h"
  66 #include "bnxt_devlink.h"
  67 #include "bnxt_debugfs.h"
  68
  69 #define BNXT_TX_TIMEOUT         (5 * HZ)
  70
  71 static const char version[] =
  72         "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
  73
  74 MODULE_LICENSE("GPL");
  75 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  76 MODULE_VERSION(DRV_MODULE_VERSION);
  77
  78 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  79 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  80 #define BNXT_RX_COPY_THRESH 256
  81
  82 #define BNXT_TX_PUSH_THRESH 164
  83
  84 enum board_idx {
  85         BCM57301,
  86         BCM57302,
  87         BCM57304,
  88         BCM57417_NPAR,
  89         BCM58700,
  90         BCM57311,
  91         BCM57312,
  92         BCM57402,
  93         BCM57404,
  94         BCM57406,
  95         BCM57402_NPAR,
  96         BCM57407,
  97         BCM57412,
  98         BCM57414,
  99         BCM57416,
 100         BCM57417,
 101         BCM57412_NPAR,
 102         BCM57314,
 103         BCM57417_SFP,
 104         BCM57416_SFP,
 105         BCM57404_NPAR,
 106         BCM57406_NPAR,
 107         BCM57407_SFP,
 108         BCM57407_NPAR,
 109         BCM57414_NPAR,
 110         BCM57416_NPAR,
 111         BCM57452,
 112         BCM57454,
 113         BCM5745x_NPAR,
 114         BCM57508,
 115         BCM58802,
 116         BCM58804,
 117         BCM58808,
 118         NETXTREME_E_VF,
 119         NETXTREME_C_VF,
 120         NETXTREME_S_VF,
 121         NETXTREME_E_P5_VF,
 122 };
 123
 124 /* indexed by enum above */
 125 static const struct {
 126         char *name;
 127 } board_info[] = {
 128         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
 129         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
 130         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 131         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
 132         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
 133         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
 134         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
 135         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 136         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 137         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 138         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 139         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 140         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 141         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 142         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 143         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 144         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 145         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 146         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 147         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 148         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 149         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 150         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 151         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 152         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 153         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 154         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 155         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 156         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 157         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 158         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 159         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 160         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 161         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 162         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 163         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 164         [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
 165 };
 166
 167 static const struct pci_device_id bnxt_pci_tbl[] = {
 168         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 169         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 170         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 171         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 172         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 173         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 174         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 175         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 176         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 177         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 178         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 179         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 180         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 181         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 182         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 183         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 184         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 185         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 186         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 187         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 188         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 189         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 190         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 191         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 192         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 193         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 194         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 195         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 196         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 197         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 198         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 199         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 200         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 201         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 202         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 203         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 204         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 205         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 206 #ifdef CONFIG_BNXT_SRIOV
 207         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 208         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 209         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 210         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 211         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 212         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 213         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 214         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 215         { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
 216         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 217 #endif
 218         { 0 }
 219 };
 220
 221 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 222
 223 static const u16 bnxt_vf_req_snif[] = {
 224         HWRM_FUNC_CFG,
 225         HWRM_FUNC_VF_CFG,
 226         HWRM_PORT_PHY_QCFG,
 227         HWRM_CFA_L2_FILTER_ALLOC,
 228 };
 229
 230 static const u16 bnxt_async_events_arr[] = {
 231         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 232         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 233         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 234         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 235         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 236 };
 237
 238 static struct workqueue_struct *bnxt_pf_wq;
 239
 240 static bool bnxt_vf_pciid(enum board_idx idx)
 241 {
 242         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 243                 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
 244 }
 245
 246 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 247 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 248 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 249
 250 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 251                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 252
 253 #define BNXT_DB_CQ(db, idx)                                             \
 254         writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
 255
 256 #define BNXT_DB_NQ_P5(db, idx)                                          \
 257         writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
 258
 259 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 260         writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
 261
 262 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 263         writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
 264
 265 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 266 {
 267         if (bp->flags & BNXT_FLAG_CHIP_P5)
 268                 BNXT_DB_NQ_P5(db, idx);
 269         else
 270                 BNXT_DB_CQ(db, idx);
 271 }
 272
 273 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 274 {
 275         if (bp->flags & BNXT_FLAG_CHIP_P5)
 276                 BNXT_DB_NQ_ARM_P5(db, idx);
 277         else
 278                 BNXT_DB_CQ_ARM(db, idx);
 279 }
 280
 281 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 282 {
 283         if (bp->flags & BNXT_FLAG_CHIP_P5)
 284                 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
 285                        db->doorbell);
 286         else
 287                 BNXT_DB_CQ(db, idx);
 288 }
 289
 290 const u16 bnxt_lhint_arr[] = {
 291         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 292         TX_BD_FLAGS_LHINT_512_TO_1023,
 293         TX_BD_FLAGS_LHINT_1024_TO_2047,
 294         TX_BD_FLAGS_LHINT_1024_TO_2047,
 295         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 296         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 297         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 298         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 299         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 300         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 301         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 302         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 303         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 304         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 305         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 306         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 307         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 308         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 309         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 310 };
 311
 312 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 313 {
 314         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 315
 316         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 317                 return 0;
 318
 319         return md_dst->u.port_info.port_id;
 320 }
 321
 322 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 323 {
 324         struct bnxt *bp = netdev_priv(dev);
 325         struct tx_bd *txbd;
 326         struct tx_bd_ext *txbd1;
 327         struct netdev_queue *txq;
 328         int i;
 329         dma_addr_t mapping;
 330         unsigned int length, pad = 0;
 331         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 332         u16 prod, last_frag;
 333         struct pci_dev *pdev = bp->pdev;
 334         struct bnxt_tx_ring_info *txr;
 335         struct bnxt_sw_tx_bd *tx_buf;
 336
 337         i = skb_get_queue_mapping(skb);
 338         if (unlikely(i >= bp->tx_nr_rings)) {
 339                 dev_kfree_skb_any(skb);
 340                 return NETDEV_TX_OK;
 341         }
 342
 343         txq = netdev_get_tx_queue(dev, i);
 344         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 345         prod = txr->tx_prod;
 346
 347         free_size = bnxt_tx_avail(bp, txr);
 348         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 349                 netif_tx_stop_queue(txq);
 350                 return NETDEV_TX_BUSY;
 351         }
 352
 353         length = skb->len;
 354         len = skb_headlen(skb);
 355         last_frag = skb_shinfo(skb)->nr_frags;
 356
 357         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 358
 359         txbd->tx_bd_opaque = prod;
 360
 361         tx_buf = &txr->tx_buf_ring[prod];
 362         tx_buf->skb = skb;
 363         tx_buf->nr_frags = last_frag;
 364
 365         vlan_tag_flags = 0;
 366         cfa_action = bnxt_xmit_get_cfa_action(skb);
 367         if (skb_vlan_tag_present(skb)) {
 368                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 369                                  skb_vlan_tag_get(skb);
 370                 /* Currently supports 8021Q, 8021AD vlan offloads
 371                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 372                  */
 373                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 374                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 375         }
 376
 377         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 378                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 379                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 380                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 381                 void __iomem *db = txr->tx_db.doorbell;
 382                 void *pdata = tx_push_buf->data;
 383                 u64 *end;
 384                 int j, push_len;
 385
 386                 /* Set COAL_NOW to be ready quickly for the next push */
 387                 tx_push->tx_bd_len_flags_type =
 388                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 389                                         TX_BD_TYPE_LONG_TX_BD |
 390                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 391                                         TX_BD_FLAGS_COAL_NOW |
 392                                         TX_BD_FLAGS_PACKET_END |
 393                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 394
 395                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 396                         tx_push1->tx_bd_hsize_lflags =
 397                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 398                 else
 399                         tx_push1->tx_bd_hsize_lflags = 0;
 400
 401                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 402                 tx_push1->tx_bd_cfa_action =
 403                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 404
 405                 end = pdata + length;
 406                 end = PTR_ALIGN(end, 8) - 1;
 407                 *end = 0;
 408
 409                 skb_copy_from_linear_data(skb, pdata, len);
 410                 pdata += len;
 411                 for (j = 0; j < last_frag; j++) {
 412                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 413                         void *fptr;
 414
 415                         fptr = skb_frag_address_safe(frag);
 416                         if (!fptr)
 417                                 goto normal_tx;
 418
 419                         memcpy(pdata, fptr, skb_frag_size(frag));
 420                         pdata += skb_frag_size(frag);
 421                 }
 422
 423                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 424                 txbd->tx_bd_haddr = txr->data_mapping;
 425                 prod = NEXT_TX(prod);
 426                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 427                 memcpy(txbd, tx_push1, sizeof(*txbd));
 428                 prod = NEXT_TX(prod);
 429                 tx_push->doorbell =
 430                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 431                 txr->tx_prod = prod;
 432
 433                 tx_buf->is_push = 1;
 434                 netdev_tx_sent_queue(txq, skb->len);
 435                 wmb();  /* Sync is_push and byte queue before pushing data */
 436
 437                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 438                 if (push_len > 16) {
 439                         __iowrite64_copy(db, tx_push_buf, 16);
 440                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 441                                          (push_len - 16) << 1);
 442                 } else {
 443                         __iowrite64_copy(db, tx_push_buf, push_len);
 444                 }
 445
 446                 goto tx_done;
 447         }
 448
 449 normal_tx:
 450         if (length < BNXT_MIN_PKT_SIZE) {
 451                 pad = BNXT_MIN_PKT_SIZE - length;
 452                 if (skb_pad(skb, pad)) {
 453                         /* SKB already freed. */
 454                         tx_buf->skb = NULL;
 455                         return NETDEV_TX_OK;
 456                 }
 457                 length = BNXT_MIN_PKT_SIZE;
 458         }
 459
 460         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 461
 462         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 463                 dev_kfree_skb_any(skb);
 464                 tx_buf->skb = NULL;
 465                 return NETDEV_TX_OK;
 466         }
 467
 468         dma_unmap_addr_set(tx_buf, mapping, mapping);
 469         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 470                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 471
 472         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 473
 474         prod = NEXT_TX(prod);
 475         txbd1 = (struct tx_bd_ext *)
 476                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 477
 478         txbd1->tx_bd_hsize_lflags = 0;
 479         if (skb_is_gso(skb)) {
 480                 u32 hdr_len;
 481
 482                 if (skb->encapsulation)
 483                         hdr_len = skb_inner_network_offset(skb) +
 484                                 skb_inner_network_header_len(skb) +
 485                                 inner_tcp_hdrlen(skb);
 486                 else
 487                         hdr_len = skb_transport_offset(skb) +
 488                                 tcp_hdrlen(skb);
 489
 490                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 491                                         TX_BD_FLAGS_T_IPID |
 492                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 493                 length = skb_shinfo(skb)->gso_size;
 494                 txbd1->tx_bd_mss = cpu_to_le32(length);
 495                 length += hdr_len;
 496         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 497                 txbd1->tx_bd_hsize_lflags =
 498                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 499                 txbd1->tx_bd_mss = 0;
 500         }
 501
 502         length >>= 9;
 503         flags |= bnxt_lhint_arr[length];
 504         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 505
 506         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 507         txbd1->tx_bd_cfa_action =
 508                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 509         for (i = 0; i < last_frag; i++) {
 510                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 511
 512                 prod = NEXT_TX(prod);
 513                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 514
 515                 len = skb_frag_size(frag);
 516                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 517                                            DMA_TO_DEVICE);
 518
 519                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 520                         goto tx_dma_error;
 521
 522                 tx_buf = &txr->tx_buf_ring[prod];
 523                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 524
 525                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 526
 527                 flags = len << TX_BD_LEN_SHIFT;
 528                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 529         }
 530
 531         flags &= ~TX_BD_LEN;
 532         txbd->tx_bd_len_flags_type =
 533                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 534                             TX_BD_FLAGS_PACKET_END);
 535
 536         netdev_tx_sent_queue(txq, skb->len);
 537
 538         /* Sync BD data before updating doorbell */
 539         wmb();
 540
 541         prod = NEXT_TX(prod);
 542         txr->tx_prod = prod;
 543
 544         if (!skb->xmit_more || netif_xmit_stopped(txq))
 545                 bnxt_db_write(bp, &txr->tx_db, prod);
 546
 547 tx_done:
 548
 549         mmiowb();
 550
 551         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 552                 if (skb->xmit_more && !tx_buf->is_push)
 553                         bnxt_db_write(bp, &txr->tx_db, prod);
 554
 555                 netif_tx_stop_queue(txq);
 556
 557                 /* netif_tx_stop_queue() must be done before checking
 558                  * tx index in bnxt_tx_avail() below, because in
 559                  * bnxt_tx_int(), we update tx index before checking for
 560                  * netif_tx_queue_stopped().
 561                  */
 562                 smp_mb();
 563                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 564                         netif_tx_wake_queue(txq);
 565         }
 566         return NETDEV_TX_OK;
 567
 568 tx_dma_error:
 569         last_frag = i;
 570
 571         /* start back at beginning and unmap skb */
 572         prod = txr->tx_prod;
 573         tx_buf = &txr->tx_buf_ring[prod];
 574         tx_buf->skb = NULL;
 575         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 576                          skb_headlen(skb), PCI_DMA_TODEVICE);
 577         prod = NEXT_TX(prod);
 578
 579         /* unmap remaining mapped pages */
 580         for (i = 0; i < last_frag; i++) {
 581                 prod = NEXT_TX(prod);
 582                 tx_buf = &txr->tx_buf_ring[prod];
 583                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 584                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 585                                PCI_DMA_TODEVICE);
 586         }
 587
 588         dev_kfree_skb_any(skb);
 589         return NETDEV_TX_OK;
 590 }
 591
 592 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 593 {
 594         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 595         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 596         u16 cons = txr->tx_cons;
 597         struct pci_dev *pdev = bp->pdev;
 598         int i;
 599         unsigned int tx_bytes = 0;
 600
 601         for (i = 0; i < nr_pkts; i++) {
 602                 struct bnxt_sw_tx_bd *tx_buf;
 603                 struct sk_buff *skb;
 604                 int j, last;
 605
 606                 tx_buf = &txr->tx_buf_ring[cons];
 607                 cons = NEXT_TX(cons);
 608                 skb = tx_buf->skb;
 609                 tx_buf->skb = NULL;
 610
 611                 if (tx_buf->is_push) {
 612                         tx_buf->is_push = 0;
 613                         goto next_tx_int;
 614                 }
 615
 616                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 617                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 618                 last = tx_buf->nr_frags;
 619
 620                 for (j = 0; j < last; j++) {
 621                         cons = NEXT_TX(cons);
 622                         tx_buf = &txr->tx_buf_ring[cons];
 623                         dma_unmap_page(
 624                                 &pdev->dev,
 625                                 dma_unmap_addr(tx_buf, mapping),
 626                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 627                                 PCI_DMA_TODEVICE);
 628                 }
 629
 630 next_tx_int:
 631                 cons = NEXT_TX(cons);
 632
 633                 tx_bytes += skb->len;
 634                 dev_kfree_skb_any(skb);
 635         }
 636
 637         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 638         txr->tx_cons = cons;
 639
 640         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 641          * before checking for netif_tx_queue_stopped().  Without the
 642          * memory barrier, there is a small possibility that bnxt_start_xmit()
 643          * will miss it and cause the queue to be stopped forever.
 644          */
 645         smp_mb();
 646
 647         if (unlikely(netif_tx_queue_stopped(txq)) &&
 648             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 649                 __netif_tx_lock(txq, smp_processor_id());
 650                 if (netif_tx_queue_stopped(txq) &&
 651                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 652                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 653                         netif_tx_wake_queue(txq);
 654                 __netif_tx_unlock(txq);
 655         }
 656 }
 657
 658 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 659                                          gfp_t gfp)
 660 {
 661         struct device *dev = &bp->pdev->dev;
 662         struct page *page;
 663
 664         page = alloc_page(gfp);
 665         if (!page)
 666                 return NULL;
 667
 668         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 669                                       DMA_ATTR_WEAK_ORDERING);
 670         if (dma_mapping_error(dev, *mapping)) {
 671                 __free_page(page);
 672                 return NULL;
 673         }
 674         *mapping += bp->rx_dma_offset;
 675         return page;
 676 }
 677
 678 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 679                                        gfp_t gfp)
 680 {
 681         u8 *data;
 682         struct pci_dev *pdev = bp->pdev;
 683
 684         data = kmalloc(bp->rx_buf_size, gfp);
 685         if (!data)
 686                 return NULL;
 687
 688         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 689                                         bp->rx_buf_use_size, bp->rx_dir,
 690                                         DMA_ATTR_WEAK_ORDERING);
 691
 692         if (dma_mapping_error(&pdev->dev, *mapping)) {
 693                 kfree(data);
 694                 data = NULL;
 695         }
 696         return data;
 697 }
 698
 699 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 700                        u16 prod, gfp_t gfp)
 701 {
 702         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 703         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 704         dma_addr_t mapping;
 705
 706         if (BNXT_RX_PAGE_MODE(bp)) {
 707                 struct page *page = __bnxt_alloc_rx_page(bp, &mapping, gfp);
 708
 709                 if (!page)
 710                         return -ENOMEM;
 711
 712                 rx_buf->data = page;
 713                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 714         } else {
 715                 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 716
 717                 if (!data)
 718                         return -ENOMEM;
 719
 720                 rx_buf->data = data;
 721                 rx_buf->data_ptr = data + bp->rx_offset;
 722         }
 723         rx_buf->mapping = mapping;
 724
 725         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 726         return 0;
 727 }
 728
 729 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 730 {
 731         u16 prod = rxr->rx_prod;
 732         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 733         struct rx_bd *cons_bd, *prod_bd;
 734
 735         prod_rx_buf = &rxr->rx_buf_ring[prod];
 736         cons_rx_buf = &rxr->rx_buf_ring[cons];
 737
 738         prod_rx_buf->data = data;
 739         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 740
 741         prod_rx_buf->mapping = cons_rx_buf->mapping;
 742
 743         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 744         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 745
 746         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 747 }
 748
 749 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 750 {
 751         u16 next, max = rxr->rx_agg_bmap_size;
 752
 753         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 754         if (next >= max)
 755                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 756         return next;
 757 }
 758
 759 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 760                                      struct bnxt_rx_ring_info *rxr,
 761                                      u16 prod, gfp_t gfp)
 762 {
 763         struct rx_bd *rxbd =
 764                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 765         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 766         struct pci_dev *pdev = bp->pdev;
 767         struct page *page;
 768         dma_addr_t mapping;
 769         u16 sw_prod = rxr->rx_sw_agg_prod;
 770         unsigned int offset = 0;
 771
 772         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 773                 page = rxr->rx_page;
 774                 if (!page) {
 775                         page = alloc_page(gfp);
 776                         if (!page)
 777                                 return -ENOMEM;
 778                         rxr->rx_page = page;
 779                         rxr->rx_page_offset = 0;
 780                 }
 781                 offset = rxr->rx_page_offset;
 782                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 783                 if (rxr->rx_page_offset == PAGE_SIZE)
 784                         rxr->rx_page = NULL;
 785                 else
 786                         get_page(page);
 787         } else {
 788                 page = alloc_page(gfp);
 789                 if (!page)
 790                         return -ENOMEM;
 791         }
 792
 793         mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 794                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
 795                                      DMA_ATTR_WEAK_ORDERING);
 796         if (dma_mapping_error(&pdev->dev, mapping)) {
 797                 __free_page(page);
 798                 return -EIO;
 799         }
 800
 801         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 802                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 803
 804         __set_bit(sw_prod, rxr->rx_agg_bmap);
 805         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 806         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 807
 808         rx_agg_buf->page = page;
 809         rx_agg_buf->offset = offset;
 810         rx_agg_buf->mapping = mapping;
 811         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 812         rxbd->rx_bd_opaque = sw_prod;
 813         return 0;
 814 }
 815
 816 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 cp_cons,
 817                                    u32 agg_bufs)
 818 {
 819         struct bnxt_napi *bnapi = cpr->bnapi;
 820         struct bnxt *bp = bnapi->bp;
 821         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 822         u16 prod = rxr->rx_agg_prod;
 823         u16 sw_prod = rxr->rx_sw_agg_prod;
 824         u32 i;
 825
 826         for (i = 0; i < agg_bufs; i++) {
 827                 u16 cons;
 828                 struct rx_agg_cmp *agg;
 829                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 830                 struct rx_bd *prod_bd;
 831                 struct page *page;
 832
 833                 agg = (struct rx_agg_cmp *)
 834                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 835                 cons = agg->rx_agg_cmp_opaque;
 836                 __clear_bit(cons, rxr->rx_agg_bmap);
 837
 838                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 839                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 840
 841                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 842                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 843                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 844
 845                 /* It is possible for sw_prod to be equal to cons, so
 846                  * set cons_rx_buf->page to NULL first.
 847                  */
 848                 page = cons_rx_buf->page;
 849                 cons_rx_buf->page = NULL;
 850                 prod_rx_buf->page = page;
 851                 prod_rx_buf->offset = cons_rx_buf->offset;
 852
 853                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 854
 855                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 856
 857                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 858                 prod_bd->rx_bd_opaque = sw_prod;
 859
 860                 prod = NEXT_RX_AGG(prod);
 861                 sw_prod = NEXT_RX_AGG(sw_prod);
 862                 cp_cons = NEXT_CMP(cp_cons);
 863         }
 864         rxr->rx_agg_prod = prod;
 865         rxr->rx_sw_agg_prod = sw_prod;
 866 }
 867
 868 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 869                                         struct bnxt_rx_ring_info *rxr,
 870                                         u16 cons, void *data, u8 *data_ptr,
 871                                         dma_addr_t dma_addr,
 872                                         unsigned int offset_and_len)
 873 {
 874         unsigned int payload = offset_and_len >> 16;
 875         unsigned int len = offset_and_len & 0xffff;
 876         struct skb_frag_struct *frag;
 877         struct page *page = data;
 878         u16 prod = rxr->rx_prod;
 879         struct sk_buff *skb;
 880         int off, err;
 881
 882         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 883         if (unlikely(err)) {
 884                 bnxt_reuse_rx_data(rxr, cons, data);
 885                 return NULL;
 886         }
 887         dma_addr -= bp->rx_dma_offset;
 888         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 889                              DMA_ATTR_WEAK_ORDERING);
 890
 891         if (unlikely(!payload))
 892                 payload = eth_get_headlen(data_ptr, len);
 893
 894         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
 895         if (!skb) {
 896                 __free_page(page);
 897                 return NULL;
 898         }
 899
 900         off = (void *)data_ptr - page_address(page);
 901         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
 902         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
 903                payload + NET_IP_ALIGN);
 904
 905         frag = &skb_shinfo(skb)->frags[0];
 906         skb_frag_size_sub(frag, payload);
 907         frag->page_offset += payload;
 908         skb->data_len -= payload;
 909         skb->tail += payload;
 910
 911         return skb;
 912 }
 913
 914 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 915                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 916                                    void *data, u8 *data_ptr,
 917                                    dma_addr_t dma_addr,
 918                                    unsigned int offset_and_len)
 919 {
 920         u16 prod = rxr->rx_prod;
 921         struct sk_buff *skb;
 922         int err;
 923
 924         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 925         if (unlikely(err)) {
 926                 bnxt_reuse_rx_data(rxr, cons, data);
 927                 return NULL;
 928         }
 929
 930         skb = build_skb(data, 0);
 931         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 932                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
 933         if (!skb) {
 934                 kfree(data);
 935                 return NULL;
 936         }
 937
 938         skb_reserve(skb, bp->rx_offset);
 939         skb_put(skb, offset_and_len & 0xffff);
 940         return skb;
 941 }
 942
 943 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
 944                                      struct bnxt_cp_ring_info *cpr,
 945                                      struct sk_buff *skb, u16 cp_cons,
 946                                      u32 agg_bufs)
 947 {
 948         struct bnxt_napi *bnapi = cpr->bnapi;
 949         struct pci_dev *pdev = bp->pdev;
 950         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 951         u16 prod = rxr->rx_agg_prod;
 952         u32 i;
 953
 954         for (i = 0; i < agg_bufs; i++) {
 955                 u16 cons, frag_len;
 956                 struct rx_agg_cmp *agg;
 957                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
 958                 struct page *page;
 959                 dma_addr_t mapping;
 960
 961                 agg = (struct rx_agg_cmp *)
 962                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 963                 cons = agg->rx_agg_cmp_opaque;
 964                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
 965                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
 966
 967                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 968                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
 969                                    cons_rx_buf->offset, frag_len);
 970                 __clear_bit(cons, rxr->rx_agg_bmap);
 971
 972                 /* It is possible for bnxt_alloc_rx_page() to allocate
 973                  * a sw_prod index that equals the cons index, so we
 974                  * need to clear the cons entry now.
 975                  */
 976                 mapping = cons_rx_buf->mapping;
 977                 page = cons_rx_buf->page;
 978                 cons_rx_buf->page = NULL;
 979
 980                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
 981                         struct skb_shared_info *shinfo;
 982                         unsigned int nr_frags;
 983
 984                         shinfo = skb_shinfo(skb);
 985                         nr_frags = --shinfo->nr_frags;
 986                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
 987
 988                         dev_kfree_skb(skb);
 989
 990                         cons_rx_buf->page = page;
 991
 992                         /* Update prod since possibly some pages have been
 993                          * allocated already.
 994                          */
 995                         rxr->rx_agg_prod = prod;
 996                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, agg_bufs - i);
 997                         return NULL;
 998                 }
 999
1000                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1001                                      PCI_DMA_FROMDEVICE,
1002                                      DMA_ATTR_WEAK_ORDERING);
1003
1004                 skb->data_len += frag_len;
1005                 skb->len += frag_len;
1006                 skb->truesize += PAGE_SIZE;
1007
1008                 prod = NEXT_RX_AGG(prod);
1009                 cp_cons = NEXT_CMP(cp_cons);
1010         }
1011         rxr->rx_agg_prod = prod;
1012         return skb;
1013 }
1014
1015 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1016                                u8 agg_bufs, u32 *raw_cons)
1017 {
1018         u16 last;
1019         struct rx_agg_cmp *agg;
1020
1021         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1022         last = RING_CMP(*raw_cons);
1023         agg = (struct rx_agg_cmp *)
1024                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1025         return RX_AGG_CMP_VALID(agg, *raw_cons);
1026 }
1027
1028 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1029                                             unsigned int len,
1030                                             dma_addr_t mapping)
1031 {
1032         struct bnxt *bp = bnapi->bp;
1033         struct pci_dev *pdev = bp->pdev;
1034         struct sk_buff *skb;
1035
1036         skb = napi_alloc_skb(&bnapi->napi, len);
1037         if (!skb)
1038                 return NULL;
1039
1040         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1041                                 bp->rx_dir);
1042
1043         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1044                len + NET_IP_ALIGN);
1045
1046         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1047                                    bp->rx_dir);
1048
1049         skb_put(skb, len);
1050         return skb;
1051 }
1052
1053 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1054                            u32 *raw_cons, void *cmp)
1055 {
1056         struct rx_cmp *rxcmp = cmp;
1057         u32 tmp_raw_cons = *raw_cons;
1058         u8 cmp_type, agg_bufs = 0;
1059
1060         cmp_type = RX_CMP_TYPE(rxcmp);
1061
1062         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1063                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1064                             RX_CMP_AGG_BUFS) >>
1065                            RX_CMP_AGG_BUFS_SHIFT;
1066         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1067                 struct rx_tpa_end_cmp *tpa_end = cmp;
1068
1069                 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1070                             RX_TPA_END_CMP_AGG_BUFS) >>
1071                            RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1072         }
1073
1074         if (agg_bufs) {
1075                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1076                         return -EBUSY;
1077         }
1078         *raw_cons = tmp_raw_cons;
1079         return 0;
1080 }
1081
1082 static void bnxt_queue_sp_work(struct bnxt *bp)
1083 {
1084         if (BNXT_PF(bp))
1085                 queue_work(bnxt_pf_wq, &bp->sp_task);
1086         else
1087                 schedule_work(&bp->sp_task);
1088 }
1089
1090 static void bnxt_cancel_sp_work(struct bnxt *bp)
1091 {
1092         if (BNXT_PF(bp))
1093                 flush_workqueue(bnxt_pf_wq);
1094         else
1095                 cancel_work_sync(&bp->sp_task);
1096 }
1097
1098 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1099 {
1100         if (!rxr->bnapi->in_reset) {
1101                 rxr->bnapi->in_reset = true;
1102                 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1103                 bnxt_queue_sp_work(bp);
1104         }
1105         rxr->rx_next_cons = 0xffff;
1106 }
1107
1108 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1109                            struct rx_tpa_start_cmp *tpa_start,
1110                            struct rx_tpa_start_cmp_ext *tpa_start1)
1111 {
1112         u8 agg_id = TPA_START_AGG_ID(tpa_start);
1113         u16 cons, prod;
1114         struct bnxt_tpa_info *tpa_info;
1115         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1116         struct rx_bd *prod_bd;
1117         dma_addr_t mapping;
1118
1119         cons = tpa_start->rx_tpa_start_cmp_opaque;
1120         prod = rxr->rx_prod;
1121         cons_rx_buf = &rxr->rx_buf_ring[cons];
1122         prod_rx_buf = &rxr->rx_buf_ring[prod];
1123         tpa_info = &rxr->rx_tpa[agg_id];
1124
1125         if (unlikely(cons != rxr->rx_next_cons)) {
1126                 bnxt_sched_reset(bp, rxr);
1127                 return;
1128         }
1129         /* Store cfa_code in tpa_info to use in tpa_end
1130          * completion processing.
1131          */
1132         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1133         prod_rx_buf->data = tpa_info->data;
1134         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1135
1136         mapping = tpa_info->mapping;
1137         prod_rx_buf->mapping = mapping;
1138
1139         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1140
1141         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1142
1143         tpa_info->data = cons_rx_buf->data;
1144         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1145         cons_rx_buf->data = NULL;
1146         tpa_info->mapping = cons_rx_buf->mapping;
1147
1148         tpa_info->len =
1149                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1150                                 RX_TPA_START_CMP_LEN_SHIFT;
1151         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1152                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1153
1154                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1155                 tpa_info->gso_type = SKB_GSO_TCPV4;
1156                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1157                 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1158                         tpa_info->gso_type = SKB_GSO_TCPV6;
1159                 tpa_info->rss_hash =
1160                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1161         } else {
1162                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1163                 tpa_info->gso_type = 0;
1164                 if (netif_msg_rx_err(bp))
1165                         netdev_warn(bp->dev, "TPA packet without valid hash\n");
1166         }
1167         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1168         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1169         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1170
1171         rxr->rx_prod = NEXT_RX(prod);
1172         cons = NEXT_RX(cons);
1173         rxr->rx_next_cons = NEXT_RX(cons);
1174         cons_rx_buf = &rxr->rx_buf_ring[cons];
1175
1176         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1177         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1178         cons_rx_buf->data = NULL;
1179 }
1180
1181 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 cp_cons,
1182                            u32 agg_bufs)
1183 {
1184         if (agg_bufs)
1185                 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, agg_bufs);
1186 }
1187
1188 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1189                                            int payload_off, int tcp_ts,
1190                                            struct sk_buff *skb)
1191 {
1192 #ifdef CONFIG_INET
1193         struct tcphdr *th;
1194         int len, nw_off;
1195         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1196         u32 hdr_info = tpa_info->hdr_info;
1197         bool loopback = false;
1198
1199         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1200         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1201         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1202
1203         /* If the packet is an internal loopback packet, the offsets will
1204          * have an extra 4 bytes.
1205          */
1206         if (inner_mac_off == 4) {
1207                 loopback = true;
1208         } else if (inner_mac_off > 4) {
1209                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1210                                             ETH_HLEN - 2));
1211
1212                 /* We only support inner iPv4/ipv6.  If we don't see the
1213                  * correct protocol ID, it must be a loopback packet where
1214                  * the offsets are off by 4.
1215                  */
1216                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1217                         loopback = true;
1218         }
1219         if (loopback) {
1220                 /* internal loopback packet, subtract all offsets by 4 */
1221                 inner_ip_off -= 4;
1222                 inner_mac_off -= 4;
1223                 outer_ip_off -= 4;
1224         }
1225
1226         nw_off = inner_ip_off - ETH_HLEN;
1227         skb_set_network_header(skb, nw_off);
1228         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1229                 struct ipv6hdr *iph = ipv6_hdr(skb);
1230
1231                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1232                 len = skb->len - skb_transport_offset(skb);
1233                 th = tcp_hdr(skb);
1234                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1235         } else {
1236                 struct iphdr *iph = ip_hdr(skb);
1237
1238                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1239                 len = skb->len - skb_transport_offset(skb);
1240                 th = tcp_hdr(skb);
1241                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1242         }
1243
1244         if (inner_mac_off) { /* tunnel */
1245                 struct udphdr *uh = NULL;
1246                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1247                                             ETH_HLEN - 2));
1248
1249                 if (proto == htons(ETH_P_IP)) {
1250                         struct iphdr *iph = (struct iphdr *)skb->data;
1251
1252                         if (iph->protocol == IPPROTO_UDP)
1253                                 uh = (struct udphdr *)(iph + 1);
1254                 } else {
1255                         struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1256
1257                         if (iph->nexthdr == IPPROTO_UDP)
1258                                 uh = (struct udphdr *)(iph + 1);
1259                 }
1260                 if (uh) {
1261                         if (uh->check)
1262                                 skb_shinfo(skb)->gso_type |=
1263                                         SKB_GSO_UDP_TUNNEL_CSUM;
1264                         else
1265                                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1266                 }
1267         }
1268 #endif
1269         return skb;
1270 }
1271
1272 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1273 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1274
1275 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1276                                            int payload_off, int tcp_ts,
1277                                            struct sk_buff *skb)
1278 {
1279 #ifdef CONFIG_INET
1280         struct tcphdr *th;
1281         int len, nw_off, tcp_opt_len = 0;
1282
1283         if (tcp_ts)
1284                 tcp_opt_len = 12;
1285
1286         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1287                 struct iphdr *iph;
1288
1289                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1290                          ETH_HLEN;
1291                 skb_set_network_header(skb, nw_off);
1292                 iph = ip_hdr(skb);
1293                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1294                 len = skb->len - skb_transport_offset(skb);
1295                 th = tcp_hdr(skb);
1296                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1297         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1298                 struct ipv6hdr *iph;
1299
1300                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1301                          ETH_HLEN;
1302                 skb_set_network_header(skb, nw_off);
1303                 iph = ipv6_hdr(skb);
1304                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1305                 len = skb->len - skb_transport_offset(skb);
1306                 th = tcp_hdr(skb);
1307                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1308         } else {
1309                 dev_kfree_skb_any(skb);
1310                 return NULL;
1311         }
1312
1313         if (nw_off) { /* tunnel */
1314                 struct udphdr *uh = NULL;
1315
1316                 if (skb->protocol == htons(ETH_P_IP)) {
1317                         struct iphdr *iph = (struct iphdr *)skb->data;
1318
1319                         if (iph->protocol == IPPROTO_UDP)
1320                                 uh = (struct udphdr *)(iph + 1);
1321                 } else {
1322                         struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1323
1324                         if (iph->nexthdr == IPPROTO_UDP)
1325                                 uh = (struct udphdr *)(iph + 1);
1326                 }
1327                 if (uh) {
1328                         if (uh->check)
1329                                 skb_shinfo(skb)->gso_type |=
1330                                         SKB_GSO_UDP_TUNNEL_CSUM;
1331                         else
1332                                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1333                 }
1334         }
1335 #endif
1336         return skb;
1337 }
1338
1339 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1340                                            struct bnxt_tpa_info *tpa_info,
1341                                            struct rx_tpa_end_cmp *tpa_end,
1342                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1343                                            struct sk_buff *skb)
1344 {
1345 #ifdef CONFIG_INET
1346         int payload_off;
1347         u16 segs;
1348
1349         segs = TPA_END_TPA_SEGS(tpa_end);
1350         if (segs == 1)
1351                 return skb;
1352
1353         NAPI_GRO_CB(skb)->count = segs;
1354         skb_shinfo(skb)->gso_size =
1355                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1356         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1357         payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1358                        RX_TPA_END_CMP_PAYLOAD_OFFSET) >>
1359                       RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT;
1360         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1361         if (likely(skb))
1362                 tcp_gro_complete(skb);
1363 #endif
1364         return skb;
1365 }
1366
1367 /* Given the cfa_code of a received packet determine which
1368  * netdev (vf-rep or PF) the packet is destined to.
1369  */
1370 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1371 {
1372         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1373
1374         /* if vf-rep dev is NULL, the must belongs to the PF */
1375         return dev ? dev : bp->dev;
1376 }
1377
1378 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1379                                            struct bnxt_cp_ring_info *cpr,
1380                                            u32 *raw_cons,
1381                                            struct rx_tpa_end_cmp *tpa_end,
1382                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1383                                            u8 *event)
1384 {
1385         struct bnxt_napi *bnapi = cpr->bnapi;
1386         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1387         u8 agg_id = TPA_END_AGG_ID(tpa_end);
1388         u8 *data_ptr, agg_bufs;
1389         u16 cp_cons = RING_CMP(*raw_cons);
1390         unsigned int len;
1391         struct bnxt_tpa_info *tpa_info;
1392         dma_addr_t mapping;
1393         struct sk_buff *skb;
1394         void *data;
1395
1396         if (unlikely(bnapi->in_reset)) {
1397                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1398
1399                 if (rc < 0)
1400                         return ERR_PTR(-EBUSY);
1401                 return NULL;
1402         }
1403
1404         tpa_info = &rxr->rx_tpa[agg_id];
1405         data = tpa_info->data;
1406         data_ptr = tpa_info->data_ptr;
1407         prefetch(data_ptr);
1408         len = tpa_info->len;
1409         mapping = tpa_info->mapping;
1410
1411         agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1412                     RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1413
1414         if (agg_bufs) {
1415                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1416                         return ERR_PTR(-EBUSY);
1417
1418                 *event |= BNXT_AGG_EVENT;
1419                 cp_cons = NEXT_CMP(cp_cons);
1420         }
1421
1422         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1423                 bnxt_abort_tpa(cpr, cp_cons, agg_bufs);
1424                 if (agg_bufs > MAX_SKB_FRAGS)
1425                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1426                                     agg_bufs, (int)MAX_SKB_FRAGS);
1427                 return NULL;
1428         }
1429
1430         if (len <= bp->rx_copy_thresh) {
1431                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1432                 if (!skb) {
1433                         bnxt_abort_tpa(cpr, cp_cons, agg_bufs);
1434                         return NULL;
1435                 }
1436         } else {
1437                 u8 *new_data;
1438                 dma_addr_t new_mapping;
1439
1440                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1441                 if (!new_data) {
1442                         bnxt_abort_tpa(cpr, cp_cons, agg_bufs);
1443                         return NULL;
1444                 }
1445
1446                 tpa_info->data = new_data;
1447                 tpa_info->data_ptr = new_data + bp->rx_offset;
1448                 tpa_info->mapping = new_mapping;
1449
1450                 skb = build_skb(data, 0);
1451                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1452                                        bp->rx_buf_use_size, bp->rx_dir,
1453                                        DMA_ATTR_WEAK_ORDERING);
1454
1455                 if (!skb) {
1456                         kfree(data);
1457                         bnxt_abort_tpa(cpr, cp_cons, agg_bufs);
1458                         return NULL;
1459                 }
1460                 skb_reserve(skb, bp->rx_offset);
1461                 skb_put(skb, len);
1462         }
1463
1464         if (agg_bufs) {
1465                 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs);
1466                 if (!skb) {
1467                         /* Page reuse already handled by bnxt_rx_pages(). */
1468                         return NULL;
1469                 }
1470         }
1471
1472         skb->protocol =
1473                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1474
1475         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1476                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1477
1478         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1479             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1480                 u16 vlan_proto = tpa_info->metadata >>
1481                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1482                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1483
1484                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1485         }
1486
1487         skb_checksum_none_assert(skb);
1488         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1489                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1490                 skb->csum_level =
1491                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1492         }
1493
1494         if (TPA_END_GRO(tpa_end))
1495                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1496
1497         return skb;
1498 }
1499
1500 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1501                              struct sk_buff *skb)
1502 {
1503         if (skb->dev != bp->dev) {
1504                 /* this packet belongs to a vf-rep */
1505                 bnxt_vf_rep_rx(bp, skb);
1506                 return;
1507         }
1508         skb_record_rx_queue(skb, bnapi->index);
1509         napi_gro_receive(&bnapi->napi, skb);
1510 }
1511
1512 /* returns the following:
1513  * 1       - 1 packet successfully received
1514  * 0       - successful TPA_START, packet not completed yet
1515  * -EBUSY  - completion ring does not have all the agg buffers yet
1516  * -ENOMEM - packet aborted due to out of memory
1517  * -EIO    - packet aborted due to hw error indicated in BD
1518  */
1519 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1520                        u32 *raw_cons, u8 *event)
1521 {
1522         struct bnxt_napi *bnapi = cpr->bnapi;
1523         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1524         struct net_device *dev = bp->dev;
1525         struct rx_cmp *rxcmp;
1526         struct rx_cmp_ext *rxcmp1;
1527         u32 tmp_raw_cons = *raw_cons;
1528         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1529         struct bnxt_sw_rx_bd *rx_buf;
1530         unsigned int len;
1531         u8 *data_ptr, agg_bufs, cmp_type;
1532         dma_addr_t dma_addr;
1533         struct sk_buff *skb;
1534         void *data;
1535         int rc = 0;
1536         u32 misc;
1537
1538         rxcmp = (struct rx_cmp *)
1539                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1540
1541         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1542         cp_cons = RING_CMP(tmp_raw_cons);
1543         rxcmp1 = (struct rx_cmp_ext *)
1544                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1545
1546         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1547                 return -EBUSY;
1548
1549         cmp_type = RX_CMP_TYPE(rxcmp);
1550
1551         prod = rxr->rx_prod;
1552
1553         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1554                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1555                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1556
1557                 *event |= BNXT_RX_EVENT;
1558                 goto next_rx_no_prod_no_len;
1559
1560         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1561                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1562                                    (struct rx_tpa_end_cmp *)rxcmp,
1563                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1564
1565                 if (IS_ERR(skb))
1566                         return -EBUSY;
1567
1568                 rc = -ENOMEM;
1569                 if (likely(skb)) {
1570                         bnxt_deliver_skb(bp, bnapi, skb);
1571                         rc = 1;
1572                 }
1573                 *event |= BNXT_RX_EVENT;
1574                 goto next_rx_no_prod_no_len;
1575         }
1576
1577         cons = rxcmp->rx_cmp_opaque;
1578         rx_buf = &rxr->rx_buf_ring[cons];
1579         data = rx_buf->data;
1580         data_ptr = rx_buf->data_ptr;
1581         if (unlikely(cons != rxr->rx_next_cons)) {
1582                 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
1583
1584                 bnxt_sched_reset(bp, rxr);
1585                 return rc1;
1586         }
1587         prefetch(data_ptr);
1588
1589         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1590         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1591
1592         if (agg_bufs) {
1593                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1594                         return -EBUSY;
1595
1596                 cp_cons = NEXT_CMP(cp_cons);
1597                 *event |= BNXT_AGG_EVENT;
1598         }
1599         *event |= BNXT_RX_EVENT;
1600
1601         rx_buf->data = NULL;
1602         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1603                 bnxt_reuse_rx_data(rxr, cons, data);
1604                 if (agg_bufs)
1605                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, agg_bufs);
1606
1607                 rc = -EIO;
1608                 goto next_rx;
1609         }
1610
1611         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1612         dma_addr = rx_buf->mapping;
1613
1614         if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1615                 rc = 1;
1616                 goto next_rx;
1617         }
1618
1619         if (len <= bp->rx_copy_thresh) {
1620                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1621                 bnxt_reuse_rx_data(rxr, cons, data);
1622                 if (!skb) {
1623                         rc = -ENOMEM;
1624                         goto next_rx;
1625                 }
1626         } else {
1627                 u32 payload;
1628
1629                 if (rx_buf->data_ptr == data_ptr)
1630                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1631                 else
1632                         payload = 0;
1633                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1634                                       payload | len);
1635                 if (!skb) {
1636                         rc = -ENOMEM;
1637                         goto next_rx;
1638                 }
1639         }
1640
1641         if (agg_bufs) {
1642                 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs);
1643                 if (!skb) {
1644                         rc = -ENOMEM;
1645                         goto next_rx;
1646                 }
1647         }
1648
1649         if (RX_CMP_HASH_VALID(rxcmp)) {
1650                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1651                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1652
1653                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1654                 if (hash_type != 1 && hash_type != 3)
1655                         type = PKT_HASH_TYPE_L3;
1656                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1657         }
1658
1659         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1660         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1661
1662         if ((rxcmp1->rx_cmp_flags2 &
1663              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1664             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1665                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1666                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1667                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1668
1669                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1670         }
1671
1672         skb_checksum_none_assert(skb);
1673         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1674                 if (dev->features & NETIF_F_RXCSUM) {
1675                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1676                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1677                 }
1678         } else {
1679                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1680                         if (dev->features & NETIF_F_RXCSUM)
1681                                 bnapi->cp_ring.rx_l4_csum_errors++;
1682                 }
1683         }
1684
1685         bnxt_deliver_skb(bp, bnapi, skb);
1686         rc = 1;
1687
1688 next_rx:
1689         rxr->rx_prod = NEXT_RX(prod);
1690         rxr->rx_next_cons = NEXT_RX(cons);
1691
1692         cpr->rx_packets += 1;
1693         cpr->rx_bytes += len;
1694
1695 next_rx_no_prod_no_len:
1696         *raw_cons = tmp_raw_cons;
1697
1698         return rc;
1699 }
1700
1701 /* In netpoll mode, if we are using a combined completion ring, we need to
1702  * discard the rx packets and recycle the buffers.
1703  */
1704 static int bnxt_force_rx_discard(struct bnxt *bp,
1705                                  struct bnxt_cp_ring_info *cpr,
1706                                  u32 *raw_cons, u8 *event)
1707 {
1708         u32 tmp_raw_cons = *raw_cons;
1709         struct rx_cmp_ext *rxcmp1;
1710         struct rx_cmp *rxcmp;
1711         u16 cp_cons;
1712         u8 cmp_type;
1713
1714         cp_cons = RING_CMP(tmp_raw_cons);
1715         rxcmp = (struct rx_cmp *)
1716                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1717
1718         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1719         cp_cons = RING_CMP(tmp_raw_cons);
1720         rxcmp1 = (struct rx_cmp_ext *)
1721                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1722
1723         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1724                 return -EBUSY;
1725
1726         cmp_type = RX_CMP_TYPE(rxcmp);
1727         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1728                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1729                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1730         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1731                 struct rx_tpa_end_cmp_ext *tpa_end1;
1732
1733                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1734                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1735                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1736         }
1737         return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1738 }
1739
1740 #define BNXT_GET_EVENT_PORT(data)       \
1741         ((data) &                       \
1742          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1743
1744 static int bnxt_async_event_process(struct bnxt *bp,
1745                                     struct hwrm_async_event_cmpl *cmpl)
1746 {
1747         u16 event_id = le16_to_cpu(cmpl->event_id);
1748
1749         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1750         switch (event_id) {
1751         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1752                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1753                 struct bnxt_link_info *link_info = &bp->link_info;
1754
1755                 if (BNXT_VF(bp))
1756                         goto async_event_process_exit;
1757
1758                 /* print unsupported speed warning in forced speed mode only */
1759                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1760                     (data1 & 0x20000)) {
1761                         u16 fw_speed = link_info->force_link_speed;
1762                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1763
1764                         if (speed != SPEED_UNKNOWN)
1765                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1766                                             speed);
1767                 }
1768                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1769         }
1770         /* fall through */
1771         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1772                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1773                 break;
1774         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1775                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1776                 break;
1777         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1778                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1779                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1780
1781                 if (BNXT_VF(bp))
1782                         break;
1783
1784                 if (bp->pf.port_id != port_id)
1785                         break;
1786
1787                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1788                 break;
1789         }
1790         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
1791                 if (BNXT_PF(bp))
1792                         goto async_event_process_exit;
1793                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
1794                 break;
1795         default:
1796                 goto async_event_process_exit;
1797         }
1798         bnxt_queue_sp_work(bp);
1799 async_event_process_exit:
1800         bnxt_ulp_async_events(bp, cmpl);
1801         return 0;
1802 }
1803
1804 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1805 {
1806         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1807         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1808         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1809                                 (struct hwrm_fwd_req_cmpl *)txcmp;
1810
1811         switch (cmpl_type) {
1812         case CMPL_BASE_TYPE_HWRM_DONE:
1813                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1814                 if (seq_id == bp->hwrm_intr_seq_id)
1815                         bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
1816                 else
1817                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1818                 break;
1819
1820         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1821                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1822
1823                 if ((vf_id < bp->pf.first_vf_id) ||
1824                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1825                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1826                                    vf_id);
1827                         return -EINVAL;
1828                 }
1829
1830                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1831                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
1832                 bnxt_queue_sp_work(bp);
1833                 break;
1834
1835         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1836                 bnxt_async_event_process(bp,
1837                                          (struct hwrm_async_event_cmpl *)txcmp);
1838
1839         default:
1840                 break;
1841         }
1842
1843         return 0;
1844 }
1845
1846 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
1847 {
1848         struct bnxt_napi *bnapi = dev_instance;
1849         struct bnxt *bp = bnapi->bp;
1850         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1851         u32 cons = RING_CMP(cpr->cp_raw_cons);
1852
1853         cpr->event_ctr++;
1854         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1855         napi_schedule(&bnapi->napi);
1856         return IRQ_HANDLED;
1857 }
1858
1859 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
1860 {
1861         u32 raw_cons = cpr->cp_raw_cons;
1862         u16 cons = RING_CMP(raw_cons);
1863         struct tx_cmp *txcmp;
1864
1865         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1866
1867         return TX_CMP_VALID(txcmp, raw_cons);
1868 }
1869
1870 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
1871 {
1872         struct bnxt_napi *bnapi = dev_instance;
1873         struct bnxt *bp = bnapi->bp;
1874         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1875         u32 cons = RING_CMP(cpr->cp_raw_cons);
1876         u32 int_status;
1877
1878         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1879
1880         if (!bnxt_has_work(bp, cpr)) {
1881                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
1882                 /* return if erroneous interrupt */
1883                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
1884                         return IRQ_NONE;
1885         }
1886
1887         /* disable ring IRQ */
1888         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
1889
1890         /* Return here if interrupt is shared and is disabled. */
1891         if (unlikely(atomic_read(&bp->intr_sem) != 0))
1892                 return IRQ_HANDLED;
1893
1894         napi_schedule(&bnapi->napi);
1895         return IRQ_HANDLED;
1896 }
1897
1898 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1899                             int budget)
1900 {
1901         struct bnxt_napi *bnapi = cpr->bnapi;
1902         u32 raw_cons = cpr->cp_raw_cons;
1903         u32 cons;
1904         int tx_pkts = 0;
1905         int rx_pkts = 0;
1906         u8 event = 0;
1907         struct tx_cmp *txcmp;
1908
1909         cpr->has_more_work = 0;
1910         while (1) {
1911                 int rc;
1912
1913                 cons = RING_CMP(raw_cons);
1914                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1915
1916                 if (!TX_CMP_VALID(txcmp, raw_cons))
1917                         break;
1918
1919                 /* The valid test of the entry must be done first before
1920                  * reading any further.
1921                  */
1922                 dma_rmb();
1923                 cpr->had_work_done = 1;
1924                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
1925                         tx_pkts++;
1926                         /* return full budget so NAPI will complete. */
1927                         if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
1928                                 rx_pkts = budget;
1929                                 raw_cons = NEXT_RAW_CMP(raw_cons);
1930                                 if (budget)
1931                                         cpr->has_more_work = 1;
1932                                 break;
1933                         }
1934                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1935                         if (likely(budget))
1936                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
1937                         else
1938                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
1939                                                            &event);
1940                         if (likely(rc >= 0))
1941                                 rx_pkts += rc;
1942                         /* Increment rx_pkts when rc is -ENOMEM to count towards
1943                          * the NAPI budget.  Otherwise, we may potentially loop
1944                          * here forever if we consistently cannot allocate
1945                          * buffers.
1946                          */
1947                         else if (rc == -ENOMEM && budget)
1948                                 rx_pkts++;
1949                         else if (rc == -EBUSY)  /* partial completion */
1950                                 break;
1951                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
1952                                      CMPL_BASE_TYPE_HWRM_DONE) ||
1953                                     (TX_CMP_TYPE(txcmp) ==
1954                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
1955                                     (TX_CMP_TYPE(txcmp) ==
1956                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
1957                         bnxt_hwrm_handler(bp, txcmp);
1958                 }
1959                 raw_cons = NEXT_RAW_CMP(raw_cons);
1960
1961                 if (rx_pkts && rx_pkts == budget) {
1962                         cpr->has_more_work = 1;
1963                         break;
1964                 }
1965         }
1966
1967         if (event & BNXT_TX_EVENT) {
1968                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
1969                 u16 prod = txr->tx_prod;
1970
1971                 /* Sync BD data before updating doorbell */
1972                 wmb();
1973
1974                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
1975         }
1976
1977         cpr->cp_raw_cons = raw_cons;
1978         bnapi->tx_pkts += tx_pkts;
1979         bnapi->events |= event;
1980         return rx_pkts;
1981 }
1982
1983 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
1984 {
1985         if (bnapi->tx_pkts) {
1986                 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
1987                 bnapi->tx_pkts = 0;
1988         }
1989
1990         if (bnapi->events & BNXT_RX_EVENT) {
1991                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1992
1993                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
1994                 if (bnapi->events & BNXT_AGG_EVENT)
1995                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
1996         }
1997         bnapi->events = 0;
1998 }
1999
2000 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2001                           int budget)
2002 {
2003         struct bnxt_napi *bnapi = cpr->bnapi;
2004         int rx_pkts;
2005
2006         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2007
2008         /* ACK completion ring before freeing tx ring and producing new
2009          * buffers in rx/agg rings to prevent overflowing the completion
2010          * ring.
2011          */
2012         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2013
2014         __bnxt_poll_work_done(bp, bnapi);
2015         return rx_pkts;
2016 }
2017
2018 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2019 {
2020         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2021         struct bnxt *bp = bnapi->bp;
2022         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2023         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2024         struct tx_cmp *txcmp;
2025         struct rx_cmp_ext *rxcmp1;
2026         u32 cp_cons, tmp_raw_cons;
2027         u32 raw_cons = cpr->cp_raw_cons;
2028         u32 rx_pkts = 0;
2029         u8 event = 0;
2030
2031         while (1) {
2032                 int rc;
2033
2034                 cp_cons = RING_CMP(raw_cons);
2035                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2036
2037                 if (!TX_CMP_VALID(txcmp, raw_cons))
2038                         break;
2039
2040                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2041                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2042                         cp_cons = RING_CMP(tmp_raw_cons);
2043                         rxcmp1 = (struct rx_cmp_ext *)
2044                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2045
2046                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2047                                 break;
2048
2049                         /* force an error to recycle the buffer */
2050                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2051                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2052
2053                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2054                         if (likely(rc == -EIO) && budget)
2055                                 rx_pkts++;
2056                         else if (rc == -EBUSY)  /* partial completion */
2057                                 break;
2058                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2059                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2060                         bnxt_hwrm_handler(bp, txcmp);
2061                 } else {
2062                         netdev_err(bp->dev,
2063                                    "Invalid completion received on special ring\n");
2064                 }
2065                 raw_cons = NEXT_RAW_CMP(raw_cons);
2066
2067                 if (rx_pkts == budget)
2068                         break;
2069         }
2070
2071         cpr->cp_raw_cons = raw_cons;
2072         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2073         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2074
2075         if (event & BNXT_AGG_EVENT)
2076                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2077
2078         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2079                 napi_complete_done(napi, rx_pkts);
2080                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2081         }
2082         return rx_pkts;
2083 }
2084
2085 static int bnxt_poll(struct napi_struct *napi, int budget)
2086 {
2087         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2088         struct bnxt *bp = bnapi->bp;
2089         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2090         int work_done = 0;
2091
2092         while (1) {
2093                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2094
2095                 if (work_done >= budget) {
2096                         if (!budget)
2097                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2098                         break;
2099                 }
2100
2101                 if (!bnxt_has_work(bp, cpr)) {
2102                         if (napi_complete_done(napi, work_done))
2103                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2104                         break;
2105                 }
2106         }
2107         if (bp->flags & BNXT_FLAG_DIM) {
2108                 struct net_dim_sample dim_sample;
2109
2110                 net_dim_sample(cpr->event_ctr,
2111                                cpr->rx_packets,
2112                                cpr->rx_bytes,
2113                                &dim_sample);
2114                 net_dim(&cpr->dim, dim_sample);
2115         }
2116         mmiowb();
2117         return work_done;
2118 }
2119
2120 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2121 {
2122         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2123         int i, work_done = 0;
2124
2125         for (i = 0; i < 2; i++) {
2126                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2127
2128                 if (cpr2) {
2129                         work_done += __bnxt_poll_work(bp, cpr2,
2130                                                       budget - work_done);
2131                         cpr->has_more_work |= cpr2->has_more_work;
2132                 }
2133         }
2134         return work_done;
2135 }
2136
2137 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2138                                  u64 dbr_type, bool all)
2139 {
2140         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2141         int i;
2142
2143         for (i = 0; i < 2; i++) {
2144                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2145                 struct bnxt_db_info *db;
2146
2147                 if (cpr2 && (all || cpr2->had_work_done)) {
2148                         db = &cpr2->cp_db;
2149                         writeq(db->db_key64 | dbr_type |
2150                                RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2151                         cpr2->had_work_done = 0;
2152                 }
2153         }
2154         __bnxt_poll_work_done(bp, bnapi);
2155 }
2156
2157 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2158 {
2159         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2160         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2161         u32 raw_cons = cpr->cp_raw_cons;
2162         struct bnxt *bp = bnapi->bp;
2163         struct nqe_cn *nqcmp;
2164         int work_done = 0;
2165         u32 cons;
2166
2167         if (cpr->has_more_work) {
2168                 cpr->has_more_work = 0;
2169                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2170                 if (cpr->has_more_work) {
2171                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, false);
2172                         return work_done;
2173                 }
2174                 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL, true);
2175                 if (napi_complete_done(napi, work_done))
2176                         BNXT_DB_NQ_ARM_P5(&cpr->cp_db, cpr->cp_raw_cons);
2177                 return work_done;
2178         }
2179         while (1) {
2180                 cons = RING_CMP(raw_cons);
2181                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2182
2183                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2184                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
2185                                              false);
2186                         cpr->cp_raw_cons = raw_cons;
2187                         if (napi_complete_done(napi, work_done))
2188                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2189                                                   cpr->cp_raw_cons);
2190                         return work_done;
2191                 }
2192
2193                 /* The valid test of the entry must be done first before
2194                  * reading any further.
2195                  */
2196                 dma_rmb();
2197
2198                 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2199                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2200                         struct bnxt_cp_ring_info *cpr2;
2201
2202                         cpr2 = cpr->cp_ring_arr[idx];
2203                         work_done += __bnxt_poll_work(bp, cpr2,
2204                                                       budget - work_done);
2205                         cpr->has_more_work = cpr2->has_more_work;
2206                 } else {
2207                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2208                 }
2209                 raw_cons = NEXT_RAW_CMP(raw_cons);
2210                 if (cpr->has_more_work)
2211                         break;
2212         }
2213         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, true);
2214         cpr->cp_raw_cons = raw_cons;
2215         return work_done;
2216 }
2217
2218 static void bnxt_free_tx_skbs(struct bnxt *bp)
2219 {
2220         int i, max_idx;
2221         struct pci_dev *pdev = bp->pdev;
2222
2223         if (!bp->tx_ring)
2224                 return;
2225
2226         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2227         for (i = 0; i < bp->tx_nr_rings; i++) {
2228                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2229                 int j;
2230
2231                 for (j = 0; j < max_idx;) {
2232                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2233                         struct sk_buff *skb = tx_buf->skb;
2234                         int k, last;
2235
2236                         if (!skb) {
2237                                 j++;
2238                                 continue;
2239                         }
2240
2241                         tx_buf->skb = NULL;
2242
2243                         if (tx_buf->is_push) {
2244                                 dev_kfree_skb(skb);
2245                                 j += 2;
2246                                 continue;
2247                         }
2248
2249                         dma_unmap_single(&pdev->dev,
2250                                          dma_unmap_addr(tx_buf, mapping),
2251                                          skb_headlen(skb),
2252                                          PCI_DMA_TODEVICE);
2253
2254                         last = tx_buf->nr_frags;
2255                         j += 2;
2256                         for (k = 0; k < last; k++, j++) {
2257                                 int ring_idx = j & bp->tx_ring_mask;
2258                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2259
2260                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2261                                 dma_unmap_page(
2262                                         &pdev->dev,
2263                                         dma_unmap_addr(tx_buf, mapping),
2264                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
2265                         }
2266                         dev_kfree_skb(skb);
2267                 }
2268                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2269         }
2270 }
2271
2272 static void bnxt_free_rx_skbs(struct bnxt *bp)
2273 {
2274         int i, max_idx, max_agg_idx;
2275         struct pci_dev *pdev = bp->pdev;
2276
2277         if (!bp->rx_ring)
2278                 return;
2279
2280         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2281         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2282         for (i = 0; i < bp->rx_nr_rings; i++) {
2283                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2284                 int j;
2285
2286                 if (rxr->rx_tpa) {
2287                         for (j = 0; j < MAX_TPA; j++) {
2288                                 struct bnxt_tpa_info *tpa_info =
2289                                                         &rxr->rx_tpa[j];
2290                                 u8 *data = tpa_info->data;
2291
2292                                 if (!data)
2293                                         continue;
2294
2295                                 dma_unmap_single_attrs(&pdev->dev,
2296                                                        tpa_info->mapping,
2297                                                        bp->rx_buf_use_size,
2298                                                        bp->rx_dir,
2299                                                        DMA_ATTR_WEAK_ORDERING);
2300
2301                                 tpa_info->data = NULL;
2302
2303                                 kfree(data);
2304                         }
2305                 }
2306
2307                 for (j = 0; j < max_idx; j++) {
2308                         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
2309                         dma_addr_t mapping = rx_buf->mapping;
2310                         void *data = rx_buf->data;
2311
2312                         if (!data)
2313                                 continue;
2314
2315                         rx_buf->data = NULL;
2316
2317                         if (BNXT_RX_PAGE_MODE(bp)) {
2318                                 mapping -= bp->rx_dma_offset;
2319                                 dma_unmap_page_attrs(&pdev->dev, mapping,
2320                                                      PAGE_SIZE, bp->rx_dir,
2321                                                      DMA_ATTR_WEAK_ORDERING);
2322                                 __free_page(data);
2323                         } else {
2324                                 dma_unmap_single_attrs(&pdev->dev, mapping,
2325                                                        bp->rx_buf_use_size,
2326                                                        bp->rx_dir,
2327                                                        DMA_ATTR_WEAK_ORDERING);
2328                                 kfree(data);
2329                         }
2330                 }
2331
2332                 for (j = 0; j < max_agg_idx; j++) {
2333                         struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2334                                 &rxr->rx_agg_ring[j];
2335                         struct page *page = rx_agg_buf->page;
2336
2337                         if (!page)
2338                                 continue;
2339
2340                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2341                                              BNXT_RX_PAGE_SIZE,
2342                                              PCI_DMA_FROMDEVICE,
2343                                              DMA_ATTR_WEAK_ORDERING);
2344
2345                         rx_agg_buf->page = NULL;
2346                         __clear_bit(j, rxr->rx_agg_bmap);
2347
2348                         __free_page(page);
2349                 }
2350                 if (rxr->rx_page) {
2351                         __free_page(rxr->rx_page);
2352                         rxr->rx_page = NULL;
2353                 }
2354         }
2355 }
2356
2357 static void bnxt_free_skbs(struct bnxt *bp)
2358 {
2359         bnxt_free_tx_skbs(bp);
2360         bnxt_free_rx_skbs(bp);
2361 }
2362
2363 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2364 {
2365         struct pci_dev *pdev = bp->pdev;
2366         int i;
2367
2368         for (i = 0; i < rmem->nr_pages; i++) {
2369                 if (!rmem->pg_arr[i])
2370                         continue;
2371
2372                 dma_free_coherent(&pdev->dev, rmem->page_size,
2373                                   rmem->pg_arr[i], rmem->dma_arr[i]);
2374
2375                 rmem->pg_arr[i] = NULL;
2376         }
2377         if (rmem->pg_tbl) {
2378                 size_t pg_tbl_size = rmem->nr_pages * 8;
2379
2380                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2381                         pg_tbl_size = rmem->page_size;
2382                 dma_free_coherent(&pdev->dev, pg_tbl_size,
2383                                   rmem->pg_tbl, rmem->pg_tbl_map);
2384                 rmem->pg_tbl = NULL;
2385         }
2386         if (rmem->vmem_size && *rmem->vmem) {
2387                 vfree(*rmem->vmem);
2388                 *rmem->vmem = NULL;
2389         }
2390 }
2391
2392 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2393 {
2394         struct pci_dev *pdev = bp->pdev;
2395         u64 valid_bit = 0;
2396         int i;
2397
2398         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2399                 valid_bit = PTU_PTE_VALID;
2400         if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2401                 size_t pg_tbl_size = rmem->nr_pages * 8;
2402
2403                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2404                         pg_tbl_size = rmem->page_size;
2405                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2406                                                   &rmem->pg_tbl_map,
2407                                                   GFP_KERNEL);
2408                 if (!rmem->pg_tbl)
2409                         return -ENOMEM;
2410         }
2411
2412         for (i = 0; i < rmem->nr_pages; i++) {
2413                 u64 extra_bits = valid_bit;
2414
2415                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2416                                                      rmem->page_size,
2417                                                      &rmem->dma_arr[i],
2418                                                      GFP_KERNEL);
2419                 if (!rmem->pg_arr[i])
2420                         return -ENOMEM;
2421
2422                 if (rmem->nr_pages > 1 || rmem->depth > 0) {
2423                         if (i == rmem->nr_pages - 2 &&
2424                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2425                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2426                         else if (i == rmem->nr_pages - 1 &&
2427                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2428                                 extra_bits |= PTU_PTE_LAST;
2429                         rmem->pg_tbl[i] =
2430                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2431                 }
2432         }
2433
2434         if (rmem->vmem_size) {
2435                 *rmem->vmem = vzalloc(rmem->vmem_size);
2436                 if (!(*rmem->vmem))
2437                         return -ENOMEM;
2438         }
2439         return 0;
2440 }
2441
2442 static void bnxt_free_rx_rings(struct bnxt *bp)
2443 {
2444         int i;
2445
2446         if (!bp->rx_ring)
2447                 return;
2448
2449         for (i = 0; i < bp->rx_nr_rings; i++) {
2450                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2451                 struct bnxt_ring_struct *ring;
2452
2453                 if (rxr->xdp_prog)
2454                         bpf_prog_put(rxr->xdp_prog);
2455
2456                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2457                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2458
2459                 kfree(rxr->rx_tpa);
2460                 rxr->rx_tpa = NULL;
2461
2462                 kfree(rxr->rx_agg_bmap);
2463                 rxr->rx_agg_bmap = NULL;
2464
2465                 ring = &rxr->rx_ring_struct;
2466                 bnxt_free_ring(bp, &ring->ring_mem);
2467
2468                 ring = &rxr->rx_agg_ring_struct;
2469                 bnxt_free_ring(bp, &ring->ring_mem);
2470         }
2471 }
2472
2473 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2474 {
2475         int i, rc, agg_rings = 0, tpa_rings = 0;
2476
2477         if (!bp->rx_ring)
2478                 return -ENOMEM;
2479
2480         if (bp->flags & BNXT_FLAG_AGG_RINGS)
2481                 agg_rings = 1;
2482
2483         if (bp->flags & BNXT_FLAG_TPA)
2484                 tpa_rings = 1;
2485
2486         for (i = 0; i < bp->rx_nr_rings; i++) {
2487                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2488                 struct bnxt_ring_struct *ring;
2489
2490                 ring = &rxr->rx_ring_struct;
2491
2492                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
2493                 if (rc < 0)
2494                         return rc;
2495
2496                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2497                 if (rc)
2498                         return rc;
2499
2500                 ring->grp_idx = i;
2501                 if (agg_rings) {
2502                         u16 mem_size;
2503
2504                         ring = &rxr->rx_agg_ring_struct;
2505                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2506                         if (rc)
2507                                 return rc;
2508
2509                         ring->grp_idx = i;
2510                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2511                         mem_size = rxr->rx_agg_bmap_size / 8;
2512                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2513                         if (!rxr->rx_agg_bmap)
2514                                 return -ENOMEM;
2515
2516                         if (tpa_rings) {
2517                                 rxr->rx_tpa = kcalloc(MAX_TPA,
2518                                                 sizeof(struct bnxt_tpa_info),
2519                                                 GFP_KERNEL);
2520                                 if (!rxr->rx_tpa)
2521                                         return -ENOMEM;
2522                         }
2523                 }
2524         }
2525         return 0;
2526 }
2527
2528 static void bnxt_free_tx_rings(struct bnxt *bp)
2529 {
2530         int i;
2531         struct pci_dev *pdev = bp->pdev;
2532
2533         if (!bp->tx_ring)
2534                 return;
2535
2536         for (i = 0; i < bp->tx_nr_rings; i++) {
2537                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2538                 struct bnxt_ring_struct *ring;
2539
2540                 if (txr->tx_push) {
2541                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
2542                                           txr->tx_push, txr->tx_push_mapping);
2543                         txr->tx_push = NULL;
2544                 }
2545
2546                 ring = &txr->tx_ring_struct;
2547
2548                 bnxt_free_ring(bp, &ring->ring_mem);
2549         }
2550 }
2551
2552 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2553 {
2554         int i, j, rc;
2555         struct pci_dev *pdev = bp->pdev;
2556
2557         bp->tx_push_size = 0;
2558         if (bp->tx_push_thresh) {
2559                 int push_size;
2560
2561                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2562                                         bp->tx_push_thresh);
2563
2564                 if (push_size > 256) {
2565                         push_size = 0;
2566                         bp->tx_push_thresh = 0;
2567                 }
2568
2569                 bp->tx_push_size = push_size;
2570         }
2571
2572         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2573                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2574                 struct bnxt_ring_struct *ring;
2575                 u8 qidx;
2576
2577                 ring = &txr->tx_ring_struct;
2578
2579                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2580                 if (rc)
2581                         return rc;
2582
2583                 ring->grp_idx = txr->bnapi->index;
2584                 if (bp->tx_push_size) {
2585                         dma_addr_t mapping;
2586
2587                         /* One pre-allocated DMA buffer to backup
2588                          * TX push operation
2589                          */
2590                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
2591                                                 bp->tx_push_size,
2592                                                 &txr->tx_push_mapping,
2593                                                 GFP_KERNEL);
2594
2595                         if (!txr->tx_push)
2596                                 return -ENOMEM;
2597
2598                         mapping = txr->tx_push_mapping +
2599                                 sizeof(struct tx_push_bd);
2600                         txr->data_mapping = cpu_to_le64(mapping);
2601
2602                         memset(txr->tx_push, 0, sizeof(struct tx_push_bd));
2603                 }
2604                 qidx = bp->tc_to_qidx[j];
2605                 ring->queue_id = bp->q_info[qidx].queue_id;
2606                 if (i < bp->tx_nr_rings_xdp)
2607                         continue;
2608                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2609                         j++;
2610         }
2611         return 0;
2612 }
2613
2614 static void bnxt_free_cp_rings(struct bnxt *bp)
2615 {
2616         int i;
2617
2618         if (!bp->bnapi)
2619                 return;
2620
2621         for (i = 0; i < bp->cp_nr_rings; i++) {
2622                 struct bnxt_napi *bnapi = bp->bnapi[i];
2623                 struct bnxt_cp_ring_info *cpr;
2624                 struct bnxt_ring_struct *ring;
2625                 int j;
2626
2627                 if (!bnapi)
2628                         continue;
2629
2630                 cpr = &bnapi->cp_ring;
2631                 ring = &cpr->cp_ring_struct;
2632
2633                 bnxt_free_ring(bp, &ring->ring_mem);
2634
2635                 for (j = 0; j < 2; j++) {
2636                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
2637
2638                         if (cpr2) {
2639                                 ring = &cpr2->cp_ring_struct;
2640                                 bnxt_free_ring(bp, &ring->ring_mem);
2641                                 kfree(cpr2);
2642                                 cpr->cp_ring_arr[j] = NULL;
2643                         }
2644                 }
2645         }
2646 }
2647
2648 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
2649 {
2650         struct bnxt_ring_mem_info *rmem;
2651         struct bnxt_ring_struct *ring;
2652         struct bnxt_cp_ring_info *cpr;
2653         int rc;
2654
2655         cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
2656         if (!cpr)
2657                 return NULL;
2658
2659         ring = &cpr->cp_ring_struct;
2660         rmem = &ring->ring_mem;
2661         rmem->nr_pages = bp->cp_nr_pages;
2662         rmem->page_size = HW_CMPD_RING_SIZE;
2663         rmem->pg_arr = (void **)cpr->cp_desc_ring;
2664         rmem->dma_arr = cpr->cp_desc_mapping;
2665         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
2666         rc = bnxt_alloc_ring(bp, rmem);
2667         if (rc) {
2668                 bnxt_free_ring(bp, rmem);
2669                 kfree(cpr);
2670                 cpr = NULL;
2671         }
2672         return cpr;
2673 }
2674
2675 static int bnxt_alloc_cp_rings(struct bnxt *bp)
2676 {
2677         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
2678         int i, rc, ulp_base_vec, ulp_msix;
2679
2680         ulp_msix = bnxt_get_ulp_msix_num(bp);
2681         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
2682         for (i = 0; i < bp->cp_nr_rings; i++) {
2683                 struct bnxt_napi *bnapi = bp->bnapi[i];
2684                 struct bnxt_cp_ring_info *cpr;
2685                 struct bnxt_ring_struct *ring;
2686
2687                 if (!bnapi)
2688                         continue;
2689
2690                 cpr = &bnapi->cp_ring;
2691                 cpr->bnapi = bnapi;
2692                 ring = &cpr->cp_ring_struct;
2693
2694                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2695                 if (rc)
2696                         return rc;
2697
2698                 if (ulp_msix && i >= ulp_base_vec)
2699                         ring->map_idx = i + ulp_msix;
2700                 else
2701                         ring->map_idx = i;
2702
2703                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2704                         continue;
2705
2706                 if (i < bp->rx_nr_rings) {
2707                         struct bnxt_cp_ring_info *cpr2 =
2708                                 bnxt_alloc_cp_sub_ring(bp);
2709
2710                         cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
2711                         if (!cpr2)
2712                                 return -ENOMEM;
2713                         cpr2->bnapi = bnapi;
2714                 }
2715                 if ((sh && i < bp->tx_nr_rings) ||
2716                     (!sh && i >= bp->rx_nr_rings)) {
2717                         struct bnxt_cp_ring_info *cpr2 =
2718                                 bnxt_alloc_cp_sub_ring(bp);
2719
2720                         cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
2721                         if (!cpr2)
2722                                 return -ENOMEM;
2723                         cpr2->bnapi = bnapi;
2724                 }
2725         }
2726         return 0;
2727 }
2728
2729 static void bnxt_init_ring_struct(struct bnxt *bp)
2730 {
2731         int i;
2732
2733         for (i = 0; i < bp->cp_nr_rings; i++) {
2734                 struct bnxt_napi *bnapi = bp->bnapi[i];
2735                 struct bnxt_ring_mem_info *rmem;
2736                 struct bnxt_cp_ring_info *cpr;
2737                 struct bnxt_rx_ring_info *rxr;
2738                 struct bnxt_tx_ring_info *txr;
2739                 struct bnxt_ring_struct *ring;
2740
2741                 if (!bnapi)
2742                         continue;
2743
2744                 cpr = &bnapi->cp_ring;
2745                 ring = &cpr->cp_ring_struct;
2746                 rmem = &ring->ring_mem;
2747                 rmem->nr_pages = bp->cp_nr_pages;
2748                 rmem->page_size = HW_CMPD_RING_SIZE;
2749                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
2750                 rmem->dma_arr = cpr->cp_desc_mapping;
2751                 rmem->vmem_size = 0;
2752
2753                 rxr = bnapi->rx_ring;
2754                 if (!rxr)
2755                         goto skip_rx;
2756
2757                 ring = &rxr->rx_ring_struct;
2758                 rmem = &ring->ring_mem;
2759                 rmem->nr_pages = bp->rx_nr_pages;
2760                 rmem->page_size = HW_RXBD_RING_SIZE;
2761                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
2762                 rmem->dma_arr = rxr->rx_desc_mapping;
2763                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
2764                 rmem->vmem = (void **)&rxr->rx_buf_ring;
2765
2766                 ring = &rxr->rx_agg_ring_struct;
2767                 rmem = &ring->ring_mem;
2768                 rmem->nr_pages = bp->rx_agg_nr_pages;
2769                 rmem->page_size = HW_RXBD_RING_SIZE;
2770                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
2771                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
2772                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
2773                 rmem->vmem = (void **)&rxr->rx_agg_ring;
2774
2775 skip_rx:
2776                 txr = bnapi->tx_ring;
2777                 if (!txr)
2778                         continue;
2779
2780                 ring = &txr->tx_ring_struct;
2781                 rmem = &ring->ring_mem;
2782                 rmem->nr_pages = bp->tx_nr_pages;
2783                 rmem->page_size = HW_RXBD_RING_SIZE;
2784                 rmem->pg_arr = (void **)txr->tx_desc_ring;
2785                 rmem->dma_arr = txr->tx_desc_mapping;
2786                 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
2787                 rmem->vmem = (void **)&txr->tx_buf_ring;
2788         }
2789 }
2790
2791 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
2792 {
2793         int i;
2794         u32 prod;
2795         struct rx_bd **rx_buf_ring;
2796
2797         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
2798         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
2799                 int j;
2800                 struct rx_bd *rxbd;
2801
2802                 rxbd = rx_buf_ring[i];
2803                 if (!rxbd)
2804                         continue;
2805
2806                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
2807                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
2808                         rxbd->rx_bd_opaque = prod;
2809                 }
2810         }
2811 }
2812
2813 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
2814 {
2815         struct net_device *dev = bp->dev;
2816         struct bnxt_rx_ring_info *rxr;
2817         struct bnxt_ring_struct *ring;
2818         u32 prod, type;
2819         int i;
2820
2821         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
2822                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
2823
2824         if (NET_IP_ALIGN == 2)
2825                 type |= RX_BD_FLAGS_SOP;
2826
2827         rxr = &bp->rx_ring[ring_nr];
2828         ring = &rxr->rx_ring_struct;
2829         bnxt_init_rxbd_pages(ring, type);
2830
2831         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
2832                 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1);
2833                 if (IS_ERR(rxr->xdp_prog)) {
2834                         int rc = PTR_ERR(rxr->xdp_prog);
2835
2836                         rxr->xdp_prog = NULL;
2837                         return rc;
2838                 }
2839         }
2840         prod = rxr->rx_prod;
2841         for (i = 0; i < bp->rx_ring_size; i++) {
2842                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
2843                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
2844                                     ring_nr, i, bp->rx_ring_size);
2845                         break;
2846                 }
2847                 prod = NEXT_RX(prod);
2848         }
2849         rxr->rx_prod = prod;
2850         ring->fw_ring_id = INVALID_HW_RING_ID;
2851
2852         ring = &rxr->rx_agg_ring_struct;
2853         ring->fw_ring_id = INVALID_HW_RING_ID;
2854
2855         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
2856                 return 0;
2857
2858         type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
2859                 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
2860
2861         bnxt_init_rxbd_pages(ring, type);
2862
2863         prod = rxr->rx_agg_prod;
2864         for (i = 0; i < bp->rx_agg_ring_size; i++) {
2865                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
2866                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
2867                                     ring_nr, i, bp->rx_ring_size);
2868                         break;
2869                 }
2870                 prod = NEXT_RX_AGG(prod);
2871         }
2872         rxr->rx_agg_prod = prod;
2873
2874         if (bp->flags & BNXT_FLAG_TPA) {
2875                 if (rxr->rx_tpa) {
2876                         u8 *data;
2877                         dma_addr_t mapping;
2878
2879                         for (i = 0; i < MAX_TPA; i++) {
2880                                 data = __bnxt_alloc_rx_data(bp, &mapping,
2881                                                             GFP_KERNEL);
2882                                 if (!data)
2883                                         return -ENOMEM;
2884
2885                                 rxr->rx_tpa[i].data = data;
2886                                 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
2887                                 rxr->rx_tpa[i].mapping = mapping;
2888                         }
2889                 } else {
2890                         netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
2891                         return -ENOMEM;
2892                 }
2893         }
2894
2895         return 0;
2896 }
2897
2898 static void bnxt_init_cp_rings(struct bnxt *bp)
2899 {
2900         int i, j;
2901
2902         for (i = 0; i < bp->cp_nr_rings; i++) {
2903                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
2904                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
2905
2906                 ring->fw_ring_id = INVALID_HW_RING_ID;
2907                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
2908                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
2909                 for (j = 0; j < 2; j++) {
2910                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
2911
2912                         if (!cpr2)
2913                                 continue;
2914
2915                         ring = &cpr2->cp_ring_struct;
2916                         ring->fw_ring_id = INVALID_HW_RING_ID;
2917                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
2918                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
2919                 }
2920         }
2921 }
2922
2923 static int bnxt_init_rx_rings(struct bnxt *bp)
2924 {
2925         int i, rc = 0;
2926
2927         if (BNXT_RX_PAGE_MODE(bp)) {
2928                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
2929                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
2930         } else {
2931                 bp->rx_offset = BNXT_RX_OFFSET;
2932                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
2933         }
2934
2935         for (i = 0; i < bp->rx_nr_rings; i++) {
2936                 rc = bnxt_init_one_rx_ring(bp, i);
2937                 if (rc)
2938                         break;
2939         }
2940
2941         return rc;
2942 }
2943
2944 static int bnxt_init_tx_rings(struct bnxt *bp)
2945 {
2946         u16 i;
2947
2948         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
2949                                    MAX_SKB_FRAGS + 1);
2950
2951         for (i = 0; i < bp->tx_nr_rings; i++) {
2952                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2953                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
2954
2955                 ring->fw_ring_id = INVALID_HW_RING_ID;
2956         }
2957
2958         return 0;
2959 }
2960
2961 static void bnxt_free_ring_grps(struct bnxt *bp)
2962 {
2963         kfree(bp->grp_info);
2964         bp->grp_info = NULL;
2965 }
2966
2967 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
2968 {
2969         int i;
2970
2971         if (irq_re_init) {
2972                 bp->grp_info = kcalloc(bp->cp_nr_rings,
2973                                        sizeof(struct bnxt_ring_grp_info),
2974                                        GFP_KERNEL);
2975                 if (!bp->grp_info)
2976                         return -ENOMEM;
2977         }
2978         for (i = 0; i < bp->cp_nr_rings; i++) {
2979                 if (irq_re_init)
2980                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
2981                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
2982                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
2983                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
2984                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
2985         }
2986         return 0;
2987 }
2988
2989 static void bnxt_free_vnics(struct bnxt *bp)
2990 {
2991         kfree(bp->vnic_info);
2992         bp->vnic_info = NULL;
2993         bp->nr_vnics = 0;
2994 }
2995
2996 static int bnxt_alloc_vnics(struct bnxt *bp)
2997 {
2998         int num_vnics = 1;
2999
3000 #ifdef CONFIG_RFS_ACCEL
3001         if (bp->flags & BNXT_FLAG_RFS)
3002                 num_vnics += bp->rx_nr_rings;
3003 #endif
3004
3005         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3006                 num_vnics++;
3007
3008         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3009                                 GFP_KERNEL);
3010         if (!bp->vnic_info)
3011                 return -ENOMEM;
3012
3013         bp->nr_vnics = num_vnics;
3014         return 0;
3015 }
3016
3017 static void bnxt_init_vnics(struct bnxt *bp)
3018 {
3019         int i;
3020
3021         for (i = 0; i < bp->nr_vnics; i++) {
3022                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3023                 int j;
3024
3025                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3026                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3027                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3028
3029                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3030
3031                 if (bp->vnic_info[i].rss_hash_key) {
3032                         if (i == 0)
3033                                 prandom_bytes(vnic->rss_hash_key,
3034                                               HW_HASH_KEY_SIZE);
3035                         else
3036                                 memcpy(vnic->rss_hash_key,
3037                                        bp->vnic_info[0].rss_hash_key,
3038                                        HW_HASH_KEY_SIZE);
3039                 }
3040         }
3041 }
3042
3043 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3044 {
3045         int pages;
3046
3047         pages = ring_size / desc_per_pg;
3048
3049         if (!pages)
3050                 return 1;
3051
3052         pages++;
3053
3054         while (pages & (pages - 1))
3055                 pages++;
3056
3057         return pages;
3058 }
3059
3060 void bnxt_set_tpa_flags(struct bnxt *bp)
3061 {
3062         bp->flags &= ~BNXT_FLAG_TPA;
3063         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3064                 return;
3065         if (bp->dev->features & NETIF_F_LRO)
3066                 bp->flags |= BNXT_FLAG_LRO;
3067         else if (bp->dev->features & NETIF_F_GRO_HW)
3068                 bp->flags |= BNXT_FLAG_GRO;
3069 }
3070
3071 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3072  * be set on entry.
3073  */
3074 void bnxt_set_ring_params(struct bnxt *bp)
3075 {
3076         u32 ring_size, rx_size, rx_space;
3077         u32 agg_factor = 0, agg_ring_size = 0;
3078
3079         /* 8 for CRC and VLAN */
3080         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3081
3082         rx_space = rx_size + NET_SKB_PAD +
3083                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3084
3085         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3086         ring_size = bp->rx_ring_size;
3087         bp->rx_agg_ring_size = 0;
3088         bp->rx_agg_nr_pages = 0;
3089
3090         if (bp->flags & BNXT_FLAG_TPA)
3091                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3092
3093         bp->flags &= ~BNXT_FLAG_JUMBO;
3094         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3095                 u32 jumbo_factor;
3096
3097                 bp->flags |= BNXT_FLAG_JUMBO;
3098                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3099                 if (jumbo_factor > agg_factor)
3100                         agg_factor = jumbo_factor;
3101         }
3102         agg_ring_size = ring_size * agg_factor;
3103
3104         if (agg_ring_size) {
3105                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3106                                                         RX_DESC_CNT);
3107                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3108                         u32 tmp = agg_ring_size;
3109
3110                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3111                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3112                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3113                                     tmp, agg_ring_size);
3114                 }
3115                 bp->rx_agg_ring_size = agg_ring_size;
3116                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3117                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3118                 rx_space = rx_size + NET_SKB_PAD +
3119                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3120         }
3121
3122         bp->rx_buf_use_size = rx_size;
3123         bp->rx_buf_size = rx_space;
3124
3125         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3126         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3127
3128         ring_size = bp->tx_ring_size;
3129         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3130         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3131
3132         ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
3133         bp->cp_ring_size = ring_size;
3134
3135         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3136         if (bp->cp_nr_pages > MAX_CP_PAGES) {
3137                 bp->cp_nr_pages = MAX_CP_PAGES;
3138                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3139                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3140                             ring_size, bp->cp_ring_size);
3141         }
3142         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3143         bp->cp_ring_mask = bp->cp_bit - 1;
3144 }
3145
3146 /* Changing allocation mode of RX rings.
3147  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3148  */
3149 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3150 {
3151         if (page_mode) {
3152                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3153                         return -EOPNOTSUPP;
3154                 bp->dev->max_mtu =
3155                         min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3156                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3157                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3158                 bp->rx_dir = DMA_BIDIRECTIONAL;
3159                 bp->rx_skb_func = bnxt_rx_page_skb;
3160                 /* Disable LRO or GRO_HW */
3161                 netdev_update_features(bp->dev);
3162         } else {
3163                 bp->dev->max_mtu = bp->max_mtu;
3164                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3165                 bp->rx_dir = DMA_FROM_DEVICE;
3166                 bp->rx_skb_func = bnxt_rx_skb;
3167         }
3168         return 0;
3169 }
3170
3171 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3172 {
3173         int i;
3174         struct bnxt_vnic_info *vnic;
3175         struct pci_dev *pdev = bp->pdev;
3176
3177         if (!bp->vnic_info)
3178                 return;
3179
3180         for (i = 0; i < bp->nr_vnics; i++) {
3181                 vnic = &bp->vnic_info[i];
3182
3183                 kfree(vnic->fw_grp_ids);
3184                 vnic->fw_grp_ids = NULL;
3185
3186                 kfree(vnic->uc_list);
3187                 vnic->uc_list = NULL;
3188
3189                 if (vnic->mc_list) {
3190                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3191                                           vnic->mc_list, vnic->mc_list_mapping);
3192                         vnic->mc_list = NULL;
3193                 }
3194
3195                 if (vnic->rss_table) {
3196                         dma_free_coherent(&pdev->dev, PAGE_SIZE,
3197                                           vnic->rss_table,
3198                                           vnic->rss_table_dma_addr);
3199                         vnic->rss_table = NULL;
3200                 }
3201
3202                 vnic->rss_hash_key = NULL;
3203                 vnic->flags = 0;
3204         }
3205 }
3206
3207 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3208 {
3209         int i, rc = 0, size;
3210         struct bnxt_vnic_info *vnic;
3211         struct pci_dev *pdev = bp->pdev;
3212         int max_rings;
3213
3214         for (i = 0; i < bp->nr_vnics; i++) {
3215                 vnic = &bp->vnic_info[i];
3216
3217                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3218                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3219
3220                         if (mem_size > 0) {
3221                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3222                                 if (!vnic->uc_list) {
3223                                         rc = -ENOMEM;
3224                                         goto out;
3225                                 }
3226                         }
3227                 }
3228
3229                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3230                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3231                         vnic->mc_list =
3232                                 dma_alloc_coherent(&pdev->dev,
3233                                                    vnic->mc_list_size,
3234                                                    &vnic->mc_list_mapping,
3235                                                    GFP_KERNEL);
3236                         if (!vnic->mc_list) {
3237                                 rc = -ENOMEM;
3238                                 goto out;
3239                         }
3240                 }
3241
3242                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3243                         goto vnic_skip_grps;
3244
3245                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3246                         max_rings = bp->rx_nr_rings;
3247                 else
3248                         max_rings = 1;
3249
3250                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3251                 if (!vnic->fw_grp_ids) {
3252                         rc = -ENOMEM;
3253                         goto out;
3254                 }
3255 vnic_skip_grps:
3256                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3257                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3258                         continue;
3259
3260                 /* Allocate rss table and hash key */
3261                 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3262                                                      &vnic->rss_table_dma_addr,
3263                                                      GFP_KERNEL);
3264                 if (!vnic->rss_table) {
3265                         rc = -ENOMEM;
3266                         goto out;
3267                 }
3268
3269                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3270
3271                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3272                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3273         }
3274         return 0;
3275
3276 out:
3277         return rc;
3278 }
3279
3280 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3281 {
3282         struct pci_dev *pdev = bp->pdev;
3283
3284         if (bp->hwrm_cmd_resp_addr) {
3285                 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3286                                   bp->hwrm_cmd_resp_dma_addr);
3287                 bp->hwrm_cmd_resp_addr = NULL;
3288         }
3289
3290         if (bp->hwrm_cmd_kong_resp_addr) {
3291                 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3292                                   bp->hwrm_cmd_kong_resp_addr,
3293                                   bp->hwrm_cmd_kong_resp_dma_addr);
3294                 bp->hwrm_cmd_kong_resp_addr = NULL;
3295         }
3296 }
3297
3298 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3299 {
3300         struct pci_dev *pdev = bp->pdev;
3301
3302         bp->hwrm_cmd_kong_resp_addr =
3303                 dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3304                                    &bp->hwrm_cmd_kong_resp_dma_addr,
3305                                    GFP_KERNEL);
3306         if (!bp->hwrm_cmd_kong_resp_addr)
3307                 return -ENOMEM;
3308
3309         return 0;
3310 }
3311
3312 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3313 {
3314         struct pci_dev *pdev = bp->pdev;
3315
3316         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3317                                                    &bp->hwrm_cmd_resp_dma_addr,
3318                                                    GFP_KERNEL);
3319         if (!bp->hwrm_cmd_resp_addr)
3320                 return -ENOMEM;
3321
3322         return 0;
3323 }
3324
3325 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3326 {
3327         if (bp->hwrm_short_cmd_req_addr) {
3328                 struct pci_dev *pdev = bp->pdev;
3329
3330                 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3331                                   bp->hwrm_short_cmd_req_addr,
3332                                   bp->hwrm_short_cmd_req_dma_addr);
3333                 bp->hwrm_short_cmd_req_addr = NULL;
3334         }
3335 }
3336
3337 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3338 {
3339         struct pci_dev *pdev = bp->pdev;
3340
3341         bp->hwrm_short_cmd_req_addr =
3342                 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3343                                    &bp->hwrm_short_cmd_req_dma_addr,
3344                                    GFP_KERNEL);
3345         if (!bp->hwrm_short_cmd_req_addr)
3346                 return -ENOMEM;
3347
3348         return 0;
3349 }
3350
3351 static void bnxt_free_port_stats(struct bnxt *bp)
3352 {
3353         struct pci_dev *pdev = bp->pdev;
3354
3355         bp->flags &= ~BNXT_FLAG_PORT_STATS;
3356         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3357
3358         if (bp->hw_rx_port_stats) {
3359                 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
3360                                   bp->hw_rx_port_stats,
3361                                   bp->hw_rx_port_stats_map);
3362                 bp->hw_rx_port_stats = NULL;
3363         }
3364
3365         if (bp->hw_tx_port_stats_ext) {
3366                 dma_free_coherent(&pdev->dev, sizeof(struct tx_port_stats_ext),
3367                                   bp->hw_tx_port_stats_ext,
3368                                   bp->hw_tx_port_stats_ext_map);
3369                 bp->hw_tx_port_stats_ext = NULL;
3370         }
3371
3372         if (bp->hw_rx_port_stats_ext) {
3373                 dma_free_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext),
3374                                   bp->hw_rx_port_stats_ext,
3375                                   bp->hw_rx_port_stats_ext_map);
3376                 bp->hw_rx_port_stats_ext = NULL;
3377         }
3378 }
3379
3380 static void bnxt_free_ring_stats(struct bnxt *bp)
3381 {
3382         struct pci_dev *pdev = bp->pdev;
3383         int size, i;
3384
3385         if (!bp->bnapi)
3386                 return;
3387
3388         size = sizeof(struct ctx_hw_stats);
3389
3390         for (i = 0; i < bp->cp_nr_rings; i++) {
3391                 struct bnxt_napi *bnapi = bp->bnapi[i];
3392                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3393
3394                 if (cpr->hw_stats) {
3395                         dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
3396                                           cpr->hw_stats_map);
3397                         cpr->hw_stats = NULL;
3398                 }
3399         }
3400 }
3401
3402 static int bnxt_alloc_stats(struct bnxt *bp)
3403 {
3404         u32 size, i;
3405         struct pci_dev *pdev = bp->pdev;
3406
3407         size = sizeof(struct ctx_hw_stats);
3408
3409         for (i = 0; i < bp->cp_nr_rings; i++) {
3410                 struct bnxt_napi *bnapi = bp->bnapi[i];
3411                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3412
3413                 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
3414                                                    &cpr->hw_stats_map,
3415                                                    GFP_KERNEL);
3416                 if (!cpr->hw_stats)
3417                         return -ENOMEM;
3418
3419                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3420         }
3421
3422         if (BNXT_PF(bp) && bp->chip_num != CHIP_NUM_58700) {
3423                 if (bp->hw_rx_port_stats)
3424                         goto alloc_ext_stats;
3425
3426                 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
3427                                          sizeof(struct tx_port_stats) + 1024;
3428
3429                 bp->hw_rx_port_stats =
3430                         dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
3431                                            &bp->hw_rx_port_stats_map,
3432                                            GFP_KERNEL);
3433                 if (!bp->hw_rx_port_stats)
3434                         return -ENOMEM;
3435
3436                 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) +
3437                                        512;
3438                 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
3439                                            sizeof(struct rx_port_stats) + 512;
3440                 bp->flags |= BNXT_FLAG_PORT_STATS;
3441
3442 alloc_ext_stats:
3443                 /* Display extended statistics only if FW supports it */
3444                 if (bp->hwrm_spec_code < 0x10804 ||
3445                     bp->hwrm_spec_code == 0x10900)
3446                         return 0;
3447
3448                 if (bp->hw_rx_port_stats_ext)
3449                         goto alloc_tx_ext_stats;
3450
3451                 bp->hw_rx_port_stats_ext =
3452                         dma_alloc_coherent(&pdev->dev,
3453                                            sizeof(struct rx_port_stats_ext),
3454                                            &bp->hw_rx_port_stats_ext_map,
3455                                            GFP_KERNEL);
3456                 if (!bp->hw_rx_port_stats_ext)
3457                         return 0;
3458
3459 alloc_tx_ext_stats:
3460                 if (bp->hw_tx_port_stats_ext)
3461                         return 0;
3462
3463                 if (bp->hwrm_spec_code >= 0x10902) {
3464                         bp->hw_tx_port_stats_ext =
3465                                 dma_alloc_coherent(&pdev->dev,
3466                                                    sizeof(struct tx_port_stats_ext),
3467                                                    &bp->hw_tx_port_stats_ext_map,
3468                                                    GFP_KERNEL);
3469                 }
3470                 bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
3471         }
3472         return 0;
3473 }
3474
3475 static void bnxt_clear_ring_indices(struct bnxt *bp)
3476 {
3477         int i;
3478
3479         if (!bp->bnapi)
3480                 return;
3481
3482         for (i = 0; i < bp->cp_nr_rings; i++) {
3483                 struct bnxt_napi *bnapi = bp->bnapi[i];
3484                 struct bnxt_cp_ring_info *cpr;
3485                 struct bnxt_rx_ring_info *rxr;
3486                 struct bnxt_tx_ring_info *txr;
3487
3488                 if (!bnapi)
3489                         continue;
3490
3491                 cpr = &bnapi->cp_ring;
3492                 cpr->cp_raw_cons = 0;
3493
3494                 txr = bnapi->tx_ring;
3495                 if (txr) {
3496                         txr->tx_prod = 0;
3497                         txr->tx_cons = 0;
3498                 }
3499
3500                 rxr = bnapi->rx_ring;
3501                 if (rxr) {
3502                         rxr->rx_prod = 0;
3503                         rxr->rx_agg_prod = 0;
3504                         rxr->rx_sw_agg_prod = 0;
3505                         rxr->rx_next_cons = 0;
3506                 }
3507         }
3508 }
3509
3510 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3511 {
3512 #ifdef CONFIG_RFS_ACCEL
3513         int i;
3514
3515         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
3516          * safe to delete the hash table.
3517          */
3518         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
3519                 struct hlist_head *head;
3520                 struct hlist_node *tmp;
3521                 struct bnxt_ntuple_filter *fltr;
3522
3523                 head = &bp->ntp_fltr_hash_tbl[i];
3524                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
3525                         hlist_del(&fltr->hash);
3526                         kfree(fltr);
3527                 }
3528         }
3529         if (irq_reinit) {
3530                 kfree(bp->ntp_fltr_bmap);
3531                 bp->ntp_fltr_bmap = NULL;
3532         }
3533         bp->ntp_fltr_count = 0;
3534 #endif
3535 }
3536
3537 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
3538 {
3539 #ifdef CONFIG_RFS_ACCEL
3540         int i, rc = 0;
3541
3542         if (!(bp->flags & BNXT_FLAG_RFS))
3543                 return 0;
3544
3545         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
3546                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
3547
3548         bp->ntp_fltr_count = 0;
3549         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
3550                                     sizeof(long),
3551                                     GFP_KERNEL);
3552
3553         if (!bp->ntp_fltr_bmap)
3554                 rc = -ENOMEM;
3555
3556         return rc;
3557 #else
3558         return 0;
3559 #endif
3560 }
3561
3562 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
3563 {
3564         bnxt_free_vnic_attributes(bp);
3565         bnxt_free_tx_rings(bp);
3566         bnxt_free_rx_rings(bp);
3567         bnxt_free_cp_rings(bp);
3568         bnxt_free_ntp_fltrs(bp, irq_re_init);
3569         if (irq_re_init) {
3570                 bnxt_free_ring_stats(bp);
3571                 bnxt_free_ring_grps(bp);
3572                 bnxt_free_vnics(bp);
3573                 kfree(bp->tx_ring_map);
3574                 bp->tx_ring_map = NULL;
3575                 kfree(bp->tx_ring);
3576                 bp->tx_ring = NULL;
3577                 kfree(bp->rx_ring);
3578                 bp->rx_ring = NULL;
3579                 kfree(bp->bnapi);
3580                 bp->bnapi = NULL;
3581         } else {
3582                 bnxt_clear_ring_indices(bp);
3583         }
3584 }
3585
3586 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
3587 {
3588         int i, j, rc, size, arr_size;
3589         void *bnapi;
3590
3591         if (irq_re_init) {
3592                 /* Allocate bnapi mem pointer array and mem block for
3593                  * all queues
3594                  */
3595                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
3596                                 bp->cp_nr_rings);
3597                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
3598                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
3599                 if (!bnapi)
3600                         return -ENOMEM;
3601
3602                 bp->bnapi = bnapi;
3603                 bnapi += arr_size;
3604                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
3605                         bp->bnapi[i] = bnapi;
3606                         bp->bnapi[i]->index = i;
3607                         bp->bnapi[i]->bp = bp;
3608                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
3609                                 struct bnxt_cp_ring_info *cpr =
3610                                         &bp->bnapi[i]->cp_ring;
3611
3612                                 cpr->cp_ring_struct.ring_mem.flags =
3613                                         BNXT_RMEM_RING_PTE_FLAG;
3614                         }
3615                 }
3616
3617                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
3618                                       sizeof(struct bnxt_rx_ring_info),
3619                                       GFP_KERNEL);
3620                 if (!bp->rx_ring)
3621                         return -ENOMEM;
3622
3623                 for (i = 0; i < bp->rx_nr_rings; i++) {
3624                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3625
3626                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
3627                                 rxr->rx_ring_struct.ring_mem.flags =
3628                                         BNXT_RMEM_RING_PTE_FLAG;
3629                                 rxr->rx_agg_ring_struct.ring_mem.flags =
3630                                         BNXT_RMEM_RING_PTE_FLAG;
3631                         }
3632                         rxr->bnapi = bp->bnapi[i];
3633                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
3634                 }
3635
3636                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
3637                                       sizeof(struct bnxt_tx_ring_info),
3638                                       GFP_KERNEL);
3639                 if (!bp->tx_ring)
3640                         return -ENOMEM;
3641
3642                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
3643                                           GFP_KERNEL);
3644
3645                 if (!bp->tx_ring_map)
3646                         return -ENOMEM;
3647
3648                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
3649                         j = 0;
3650                 else
3651                         j = bp->rx_nr_rings;
3652
3653                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
3654                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3655
3656                         if (bp->flags & BNXT_FLAG_CHIP_P5)
3657                                 txr->tx_ring_struct.ring_mem.flags =
3658                                         BNXT_RMEM_RING_PTE_FLAG;
3659                         txr->bnapi = bp->bnapi[j];
3660                         bp->bnapi[j]->tx_ring = txr;
3661                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
3662                         if (i >= bp->tx_nr_rings_xdp) {
3663                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
3664                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
3665                         } else {
3666                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
3667                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
3668                         }
3669                 }
3670
3671                 rc = bnxt_alloc_stats(bp);
3672                 if (rc)
3673                         goto alloc_mem_err;
3674
3675                 rc = bnxt_alloc_ntp_fltrs(bp);
3676                 if (rc)
3677                         goto alloc_mem_err;
3678
3679                 rc = bnxt_alloc_vnics(bp);
3680                 if (rc)
3681                         goto alloc_mem_err;
3682         }
3683
3684         bnxt_init_ring_struct(bp);
3685
3686         rc = bnxt_alloc_rx_rings(bp);
3687         if (rc)
3688                 goto alloc_mem_err;
3689
3690         rc = bnxt_alloc_tx_rings(bp);
3691         if (rc)
3692                 goto alloc_mem_err;
3693
3694         rc = bnxt_alloc_cp_rings(bp);
3695         if (rc)
3696                 goto alloc_mem_err;
3697
3698         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
3699                                   BNXT_VNIC_UCAST_FLAG;
3700         rc = bnxt_alloc_vnic_attributes(bp);
3701         if (rc)
3702                 goto alloc_mem_err;
3703         return 0;
3704
3705 alloc_mem_err:
3706         bnxt_free_mem(bp, true);
3707         return rc;
3708 }
3709
3710 static void bnxt_disable_int(struct bnxt *bp)
3711 {
3712         int i;
3713
3714         if (!bp->bnapi)
3715                 return;
3716
3717         for (i = 0; i < bp->cp_nr_rings; i++) {
3718                 struct bnxt_napi *bnapi = bp->bnapi[i];
3719                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3720                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3721
3722                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
3723                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
3724         }
3725 }
3726
3727 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
3728 {
3729         struct bnxt_napi *bnapi = bp->bnapi[n];
3730         struct bnxt_cp_ring_info *cpr;
3731
3732         cpr = &bnapi->cp_ring;
3733         return cpr->cp_ring_struct.map_idx;
3734 }
3735
3736 static void bnxt_disable_int_sync(struct bnxt *bp)
3737 {
3738         int i;
3739
3740         atomic_inc(&bp->intr_sem);
3741
3742         bnxt_disable_int(bp);
3743         for (i = 0; i < bp->cp_nr_rings; i++) {
3744                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
3745
3746                 synchronize_irq(bp->irq_tbl[map_idx].vector);
3747         }
3748 }
3749
3750 static void bnxt_enable_int(struct bnxt *bp)
3751 {
3752         int i;
3753
3754         atomic_set(&bp->intr_sem, 0);
3755         for (i = 0; i < bp->cp_nr_rings; i++) {
3756                 struct bnxt_napi *bnapi = bp->bnapi[i];
3757                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3758
3759                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
3760         }
3761 }
3762
3763 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
3764                             u16 cmpl_ring, u16 target_id)
3765 {
3766         struct input *req = request;
3767
3768         req->req_type = cpu_to_le16(req_type);
3769         req->cmpl_ring = cpu_to_le16(cmpl_ring);
3770         req->target_id = cpu_to_le16(target_id);
3771         if (bnxt_kong_hwrm_message(bp, req))
3772                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
3773         else
3774                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
3775 }
3776
3777 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
3778                                  int timeout, bool silent)
3779 {
3780         int i, intr_process, rc, tmo_count;
3781         struct input *req = msg;
3782         u32 *data = msg;
3783         __le32 *resp_len;
3784         u8 *valid;
3785         u16 cp_ring_id, len = 0;
3786         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
3787         u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
3788         struct hwrm_short_input short_input = {0};
3789         u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
3790         u8 *resp_addr = (u8 *)bp->hwrm_cmd_resp_addr;
3791         u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
3792         u16 dst = BNXT_HWRM_CHNL_CHIMP;
3793
3794         if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
3795                 if (msg_len > bp->hwrm_max_ext_req_len ||
3796                     !bp->hwrm_short_cmd_req_addr)
3797                         return -EINVAL;
3798         }
3799
3800         if (bnxt_hwrm_kong_chnl(bp, req)) {
3801                 dst = BNXT_HWRM_CHNL_KONG;
3802                 bar_offset = BNXT_GRCPF_REG_KONG_COMM;
3803                 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
3804                 resp = bp->hwrm_cmd_kong_resp_addr;
3805                 resp_addr = (u8 *)bp->hwrm_cmd_kong_resp_addr;
3806         }
3807
3808         memset(resp, 0, PAGE_SIZE);
3809         cp_ring_id = le16_to_cpu(req->cmpl_ring);
3810         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
3811
3812         req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
3813         /* currently supports only one outstanding message */
3814         if (intr_process)
3815                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
3816
3817         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
3818             msg_len > BNXT_HWRM_MAX_REQ_LEN) {
3819                 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
3820                 u16 max_msg_len;
3821
3822                 /* Set boundary for maximum extended request length for short
3823                  * cmd format. If passed up from device use the max supported
3824                  * internal req length.
3825                  */
3826                 max_msg_len = bp->hwrm_max_ext_req_len;
3827
3828                 memcpy(short_cmd_req, req, msg_len);
3829                 if (msg_len < max_msg_len)
3830                         memset(short_cmd_req + msg_len, 0,
3831                                max_msg_len - msg_len);
3832
3833                 short_input.req_type = req->req_type;
3834                 short_input.signature =
3835                                 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
3836                 short_input.size = cpu_to_le16(msg_len);
3837                 short_input.req_addr =
3838                         cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
3839
3840                 data = (u32 *)&short_input;
3841                 msg_len = sizeof(short_input);
3842
3843                 /* Sync memory write before updating doorbell */
3844                 wmb();
3845
3846                 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
3847         }
3848
3849         /* Write request msg to hwrm channel */
3850         __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
3851
3852         for (i = msg_len; i < max_req_len; i += 4)
3853                 writel(0, bp->bar0 + bar_offset + i);
3854
3855         /* Ring channel doorbell */
3856         writel(1, bp->bar0 + doorbell_offset);
3857
3858         if (!timeout)
3859                 timeout = DFLT_HWRM_CMD_TIMEOUT;
3860         /* convert timeout to usec */
3861         timeout *= 1000;
3862
3863         i = 0;
3864         /* Short timeout for the first few iterations:
3865          * number of loops = number of loops for short timeout +
3866          * number of loops for standard timeout.
3867          */
3868         tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
3869         timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
3870         tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
3871         resp_len = (__le32 *)(resp_addr + HWRM_RESP_LEN_OFFSET);
3872
3873         if (intr_process) {
3874                 u16 seq_id = bp->hwrm_intr_seq_id;
3875
3876                 /* Wait until hwrm response cmpl interrupt is processed */
3877                 while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
3878                        i++ < tmo_count) {
3879                         /* on first few passes, just barely sleep */
3880                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
3881                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
3882                                              HWRM_SHORT_MAX_TIMEOUT);
3883                         else
3884                                 usleep_range(HWRM_MIN_TIMEOUT,
3885                                              HWRM_MAX_TIMEOUT);
3886                 }
3887
3888                 if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
3889                         netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
3890                                    le16_to_cpu(req->req_type));
3891                         return -1;
3892                 }
3893                 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
3894                       HWRM_RESP_LEN_SFT;
3895                 valid = resp_addr + len - 1;
3896         } else {
3897                 int j;
3898
3899                 /* Check if response len is updated */
3900                 for (i = 0; i < tmo_count; i++) {
3901                         len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
3902                               HWRM_RESP_LEN_SFT;
3903                         if (len)
3904                                 break;
3905                         /* on first few passes, just barely sleep */
3906                         if (i < DFLT_HWRM_CMD_TIMEOUT)
3907                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
3908                                              HWRM_SHORT_MAX_TIMEOUT);
3909                         else
3910                                 usleep_range(HWRM_MIN_TIMEOUT,
3911                                              HWRM_MAX_TIMEOUT);
3912                 }
3913
3914                 if (i >= tmo_count) {
3915                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
3916                                    HWRM_TOTAL_TIMEOUT(i),
3917                                    le16_to_cpu(req->req_type),
3918                                    le16_to_cpu(req->seq_id), len);
3919                         return -1;
3920                 }
3921
3922                 /* Last byte of resp contains valid bit */
3923                 valid = resp_addr + len - 1;
3924                 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
3925                         /* make sure we read from updated DMA memory */
3926                         dma_rmb();
3927                         if (*valid)
3928                                 break;
3929                         udelay(1);
3930                 }
3931
3932                 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
3933                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
3934                                    HWRM_TOTAL_TIMEOUT(i),
3935                                    le16_to_cpu(req->req_type),
3936                                    le16_to_cpu(req->seq_id), len, *valid);
3937                         return -1;
3938                 }
3939         }
3940
3941         /* Zero valid bit for compatibility.  Valid bit in an older spec
3942          * may become a new field in a newer spec.  We must make sure that
3943          * a new field not implemented by old spec will read zero.
3944          */
3945         *valid = 0;
3946         rc = le16_to_cpu(resp->error_code);
3947         if (rc && !silent)
3948                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
3949                            le16_to_cpu(resp->req_type),
3950                            le16_to_cpu(resp->seq_id), rc);
3951         return rc;
3952 }
3953
3954 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
3955 {
3956         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
3957 }
3958
3959 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
3960                               int timeout)
3961 {
3962         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
3963 }
3964
3965 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
3966 {
3967         int rc;
3968
3969         mutex_lock(&bp->hwrm_cmd_lock);
3970         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
3971         mutex_unlock(&bp->hwrm_cmd_lock);
3972         return rc;
3973 }
3974
3975 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
3976                              int timeout)
3977 {
3978         int rc;
3979
3980         mutex_lock(&bp->hwrm_cmd_lock);
3981         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
3982         mutex_unlock(&bp->hwrm_cmd_lock);
3983         return rc;
3984 }
3985
3986 int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap,
3987                                      int bmap_size)
3988 {
3989         struct hwrm_func_drv_rgtr_input req = {0};
3990         DECLARE_BITMAP(async_events_bmap, 256);
3991         u32 *events = (u32 *)async_events_bmap;
3992         int i;
3993
3994         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
3995
3996         req.enables =
3997                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
3998
3999         memset(async_events_bmap, 0, sizeof(async_events_bmap));
4000         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
4001                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4002
4003         if (bmap && bmap_size) {
4004                 for (i = 0; i < bmap_size; i++) {
4005                         if (test_bit(i, bmap))
4006                                 __set_bit(i, async_events_bmap);
4007                 }
4008         }
4009
4010         for (i = 0; i < 8; i++)
4011                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4012
4013         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4014 }
4015
4016 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
4017 {
4018         struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
4019         struct hwrm_func_drv_rgtr_input req = {0};
4020         int rc;
4021
4022         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4023
4024         req.enables =
4025                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4026                             FUNC_DRV_RGTR_REQ_ENABLES_VER);
4027
4028         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4029         req.flags = cpu_to_le32(FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE);
4030         req.ver_maj_8b = DRV_VER_MAJ;
4031         req.ver_min_8b = DRV_VER_MIN;
4032         req.ver_upd_8b = DRV_VER_UPD;
4033         req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4034         req.ver_min = cpu_to_le16(DRV_VER_MIN);
4035         req.ver_upd = cpu_to_le16(DRV_VER_UPD);
4036
4037         if (BNXT_PF(bp)) {
4038                 u32 data[8];
4039                 int i;
4040
4041                 memset(data, 0, sizeof(data));
4042                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4043                         u16 cmd = bnxt_vf_req_snif[i];
4044                         unsigned int bit, idx;
4045
4046                         idx = cmd / 32;
4047                         bit = cmd % 32;
4048                         data[idx] |= 1 << bit;
4049                 }
4050
4051                 for (i = 0; i < 8; i++)
4052                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4053
4054                 req.enables |=
4055                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4056         }
4057
4058         if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4059                 req.flags |= cpu_to_le32(
4060                         FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4061
4062         mutex_lock(&bp->hwrm_cmd_lock);
4063         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4064         if (rc)
4065                 rc = -EIO;
4066         else if (resp->flags &
4067                  cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4068                 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4069         mutex_unlock(&bp->hwrm_cmd_lock);
4070         return rc;
4071 }
4072
4073 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4074 {
4075         struct hwrm_func_drv_unrgtr_input req = {0};
4076
4077         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4078         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4079 }
4080
4081 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4082 {
4083         u32 rc = 0;
4084         struct hwrm_tunnel_dst_port_free_input req = {0};
4085
4086         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4087         req.tunnel_type = tunnel_type;
4088
4089         switch (tunnel_type) {
4090         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4091                 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
4092                 break;
4093         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4094                 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
4095                 break;
4096         default:
4097                 break;
4098         }
4099
4100         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4101         if (rc)
4102                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4103                            rc);
4104         return rc;
4105 }
4106
4107 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4108                                            u8 tunnel_type)
4109 {
4110         u32 rc = 0;
4111         struct hwrm_tunnel_dst_port_alloc_input req = {0};
4112         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4113
4114         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4115
4116         req.tunnel_type = tunnel_type;
4117         req.tunnel_dst_port_val = port;
4118
4119         mutex_lock(&bp->hwrm_cmd_lock);
4120         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4121         if (rc) {
4122                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4123                            rc);
4124                 goto err_out;
4125         }
4126
4127         switch (tunnel_type) {
4128         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4129                 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
4130                 break;
4131         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4132                 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
4133                 break;
4134         default:
4135                 break;
4136         }
4137
4138 err_out:
4139         mutex_unlock(&bp->hwrm_cmd_lock);
4140         return rc;
4141 }
4142
4143 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4144 {
4145         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4146         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4147
4148         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4149         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4150
4151         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4152         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4153         req.mask = cpu_to_le32(vnic->rx_mask);
4154         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4155 }
4156
4157 #ifdef CONFIG_RFS_ACCEL
4158 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4159                                             struct bnxt_ntuple_filter *fltr)
4160 {
4161         struct hwrm_cfa_ntuple_filter_free_input req = {0};
4162
4163         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4164         req.ntuple_filter_id = fltr->filter_id;
4165         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4166 }
4167
4168 #define BNXT_NTP_FLTR_FLAGS                                     \
4169         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
4170          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
4171          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
4172          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
4173          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
4174          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
4175          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
4176          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
4177          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
4178          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
4179          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
4180          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
4181          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
4182          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4183
4184 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
4185                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4186
4187 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4188                                              struct bnxt_ntuple_filter *fltr)
4189 {
4190         struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1];
4191         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4192         struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4193         struct flow_keys *keys = &fltr->fkeys;
4194         int rc = 0;
4195
4196         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4197         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4198
4199         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4200
4201         req.ethertype = htons(ETH_P_IP);
4202         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4203         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4204         req.ip_protocol = keys->basic.ip_proto;
4205
4206         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4207                 int i;
4208
4209                 req.ethertype = htons(ETH_P_IPV6);
4210                 req.ip_addr_type =
4211                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4212                 *(struct in6_addr *)&req.src_ipaddr[0] =
4213                         keys->addrs.v6addrs.src;
4214                 *(struct in6_addr *)&req.dst_ipaddr[0] =
4215                         keys->addrs.v6addrs.dst;
4216                 for (i = 0; i < 4; i++) {
4217                         req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4218                         req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4219                 }
4220         } else {
4221                 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4222                 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4223                 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4224                 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4225         }
4226         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4227                 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4228                 req.tunnel_type =
4229                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4230         }
4231
4232         req.src_port = keys->ports.src;
4233         req.src_port_mask = cpu_to_be16(0xffff);
4234         req.dst_port = keys->ports.dst;
4235         req.dst_port_mask = cpu_to_be16(0xffff);
4236
4237         req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4238         mutex_lock(&bp->hwrm_cmd_lock);
4239         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4240         if (!rc) {
4241                 resp = bnxt_get_hwrm_resp_addr(bp, &req);
4242                 fltr->filter_id = resp->ntuple_filter_id;
4243         }
4244         mutex_unlock(&bp->hwrm_cmd_lock);
4245         return rc;
4246 }
4247 #endif
4248
4249 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4250                                      u8 *mac_addr)
4251 {
4252         u32 rc = 0;
4253         struct hwrm_cfa_l2_filter_alloc_input req = {0};
4254         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4255
4256         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4257         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4258         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4259                 req.flags |=
4260                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4261         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4262         req.enables =
4263                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4264                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4265                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4266         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4267         req.l2_addr_mask[0] = 0xff;
4268         req.l2_addr_mask[1] = 0xff;
4269         req.l2_addr_mask[2] = 0xff;
4270         req.l2_addr_mask[3] = 0xff;
4271         req.l2_addr_mask[4] = 0xff;
4272         req.l2_addr_mask[5] = 0xff;
4273
4274         mutex_lock(&bp->hwrm_cmd_lock);
4275         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4276         if (!rc)
4277                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4278                                                         resp->l2_filter_id;
4279         mutex_unlock(&bp->hwrm_cmd_lock);
4280         return rc;
4281 }
4282
4283 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4284 {
4285         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4286         int rc = 0;
4287
4288         /* Any associated ntuple filters will also be cleared by firmware. */
4289         mutex_lock(&bp->hwrm_cmd_lock);
4290         for (i = 0; i < num_of_vnics; i++) {
4291                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4292
4293                 for (j = 0; j < vnic->uc_filter_count; j++) {
4294                         struct hwrm_cfa_l2_filter_free_input req = {0};
4295
4296                         bnxt_hwrm_cmd_hdr_init(bp, &req,
4297                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
4298
4299                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
4300
4301                         rc = _hwrm_send_message(bp, &req, sizeof(req),
4302                                                 HWRM_CMD_TIMEOUT);
4303                 }
4304                 vnic->uc_filter_count = 0;
4305         }
4306         mutex_unlock(&bp->hwrm_cmd_lock);
4307
4308         return rc;
4309 }
4310
4311 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4312 {
4313         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4314         struct hwrm_vnic_tpa_cfg_input req = {0};
4315
4316         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4317                 return 0;
4318
4319         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4320
4321         if (tpa_flags) {
4322                 u16 mss = bp->dev->mtu - 40;
4323                 u32 nsegs, n, segs = 0, flags;
4324
4325                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4326                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4327                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4328                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4329                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4330                 if (tpa_flags & BNXT_FLAG_GRO)
4331                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4332
4333                 req.flags = cpu_to_le32(flags);
4334
4335                 req.enables =
4336                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
4337                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4338                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
4339
4340                 /* Number of segs are log2 units, and first packet is not
4341                  * included as part of this units.
4342                  */
4343                 if (mss <= BNXT_RX_PAGE_SIZE) {
4344                         n = BNXT_RX_PAGE_SIZE / mss;
4345                         nsegs = (MAX_SKB_FRAGS - 1) * n;
4346                 } else {
4347                         n = mss / BNXT_RX_PAGE_SIZE;
4348                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
4349                                 n++;
4350                         nsegs = (MAX_SKB_FRAGS - n) / n;
4351                 }
4352
4353                 segs = ilog2(nsegs);
4354                 req.max_agg_segs = cpu_to_le16(segs);
4355                 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX);
4356
4357                 req.min_agg_len = cpu_to_le32(512);
4358         }
4359         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4360
4361         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4362 }
4363
4364 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4365 {
4366         struct bnxt_ring_grp_info *grp_info;
4367
4368         grp_info = &bp->grp_info[ring->grp_idx];
4369         return grp_info->cp_fw_ring_id;
4370 }
4371
4372 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4373 {
4374         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4375                 struct bnxt_napi *bnapi = rxr->bnapi;
4376                 struct bnxt_cp_ring_info *cpr;
4377
4378                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4379                 return cpr->cp_ring_struct.fw_ring_id;
4380         } else {
4381                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4382         }
4383 }
4384
4385 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4386 {
4387         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4388                 struct bnxt_napi *bnapi = txr->bnapi;
4389                 struct bnxt_cp_ring_info *cpr;
4390
4391                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4392                 return cpr->cp_ring_struct.fw_ring_id;
4393         } else {
4394                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4395         }
4396 }
4397
4398 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
4399 {
4400         u32 i, j, max_rings;
4401         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4402         struct hwrm_vnic_rss_cfg_input req = {0};
4403
4404         if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
4405             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
4406                 return 0;
4407
4408         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4409         if (set_rss) {
4410                 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
4411                 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
4412                 if (vnic->flags & BNXT_VNIC_RSS_FLAG) {
4413                         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4414                                 max_rings = bp->rx_nr_rings - 1;
4415                         else
4416                                 max_rings = bp->rx_nr_rings;
4417                 } else {
4418                         max_rings = 1;
4419                 }
4420
4421                 /* Fill the RSS indirection table with ring group ids */
4422                 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
4423                         if (j == max_rings)
4424                                 j = 0;
4425                         vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
4426                 }
4427
4428                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4429                 req.hash_key_tbl_addr =
4430                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
4431         }
4432         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
4433         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4434 }
4435
4436 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
4437 {
4438         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4439         u32 i, j, k, nr_ctxs, max_rings = bp->rx_nr_rings;
4440         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
4441         struct hwrm_vnic_rss_cfg_input req = {0};
4442
4443         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4444         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4445         if (!set_rss) {
4446                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4447                 return 0;
4448         }
4449         req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
4450         req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
4451         req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4452         req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
4453         nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
4454         for (i = 0, k = 0; i < nr_ctxs; i++) {
4455                 __le16 *ring_tbl = vnic->rss_table;
4456                 int rc;
4457
4458                 req.ring_table_pair_index = i;
4459                 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
4460                 for (j = 0; j < 64; j++) {
4461                         u16 ring_id;
4462
4463                         ring_id = rxr->rx_ring_struct.fw_ring_id;
4464                         *ring_tbl++ = cpu_to_le16(ring_id);
4465                         ring_id = bnxt_cp_ring_for_rx(bp, rxr);
4466                         *ring_tbl++ = cpu_to_le16(ring_id);
4467                         rxr++;
4468                         k++;
4469                         if (k == max_rings) {
4470                                 k = 0;
4471                                 rxr = &bp->rx_ring[0];
4472                         }
4473                 }
4474                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4475                 if (rc)
4476                         return -EIO;
4477         }
4478         return 0;
4479 }
4480
4481 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
4482 {
4483         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4484         struct hwrm_vnic_plcmodes_cfg_input req = {0};
4485
4486         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
4487         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
4488                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
4489                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
4490         req.enables =
4491                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
4492                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
4493         /* thresholds not implemented in firmware yet */
4494         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
4495         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
4496         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4497         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4498 }
4499
4500 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
4501                                         u16 ctx_idx)
4502 {
4503         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
4504
4505         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
4506         req.rss_cos_lb_ctx_id =
4507                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
4508
4509         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4510         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
4511 }
4512
4513 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
4514 {
4515         int i, j;
4516
4517         for (i = 0; i < bp->nr_vnics; i++) {
4518                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4519
4520                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
4521                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
4522                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
4523                 }
4524         }
4525         bp->rsscos_nr_ctxs = 0;
4526 }
4527
4528 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
4529 {
4530         int rc;
4531         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
4532         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
4533                                                 bp->hwrm_cmd_resp_addr;
4534
4535         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
4536                                -1);
4537
4538         mutex_lock(&bp->hwrm_cmd_lock);
4539         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4540         if (!rc)
4541                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
4542                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
4543         mutex_unlock(&bp->hwrm_cmd_lock);
4544
4545         return rc;
4546 }
4547
4548 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
4549 {
4550         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
4551                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
4552         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
4553 }
4554
4555 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
4556 {
4557         unsigned int ring = 0, grp_idx;
4558         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4559         struct hwrm_vnic_cfg_input req = {0};
4560         u16 def_vlan = 0;
4561
4562         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
4563
4564         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4565                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
4566
4567                 req.default_rx_ring_id =
4568                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
4569                 req.default_cmpl_ring_id =
4570                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
4571                 req.enables =
4572                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
4573                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
4574                 goto vnic_mru;
4575         }
4576         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
4577         /* Only RSS support for now TBD: COS & LB */
4578         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
4579                 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
4580                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
4581                                            VNIC_CFG_REQ_ENABLES_MRU);
4582         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
4583                 req.rss_rule =
4584                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
4585                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
4586                                            VNIC_CFG_REQ_ENABLES_MRU);
4587                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
4588         } else {
4589                 req.rss_rule = cpu_to_le16(0xffff);
4590         }
4591
4592         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
4593             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
4594                 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
4595                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
4596         } else {
4597                 req.cos_rule = cpu_to_le16(0xffff);
4598         }
4599
4600         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4601                 ring = 0;
4602         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
4603                 ring = vnic_id - 1;
4604         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
4605                 ring = bp->rx_nr_rings - 1;
4606
4607         grp_idx = bp->rx_ring[ring].bnapi->index;
4608         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
4609         req.lb_rule = cpu_to_le16(0xffff);
4610 vnic_mru:
4611         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
4612                               VLAN_HLEN);
4613
4614         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4615 #ifdef CONFIG_BNXT_SRIOV
4616         if (BNXT_VF(bp))
4617                 def_vlan = bp->vf.vlan;
4618 #endif
4619         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
4620                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
4621         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
4622                 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
4623
4624         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4625 }
4626
4627 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
4628 {
4629         u32 rc = 0;
4630
4631         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
4632                 struct hwrm_vnic_free_input req = {0};
4633
4634                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
4635                 req.vnic_id =
4636                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
4637
4638                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4639                 if (rc)
4640                         return rc;
4641                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
4642         }
4643         return rc;
4644 }
4645
4646 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
4647 {
4648         u16 i;
4649
4650         for (i = 0; i < bp->nr_vnics; i++)
4651                 bnxt_hwrm_vnic_free_one(bp, i);
4652 }
4653
4654 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
4655                                 unsigned int start_rx_ring_idx,
4656                                 unsigned int nr_rings)
4657 {
4658         int rc = 0;
4659         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
4660         struct hwrm_vnic_alloc_input req = {0};
4661         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4662         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4663
4664         if (bp->flags & BNXT_FLAG_CHIP_P5)
4665                 goto vnic_no_ring_grps;
4666
4667         /* map ring groups to this vnic */
4668         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
4669                 grp_idx = bp->rx_ring[i].bnapi->index;
4670                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
4671                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
4672                                    j, nr_rings);
4673                         break;
4674                 }
4675                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
4676         }
4677
4678 vnic_no_ring_grps:
4679         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
4680                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
4681         if (vnic_id == 0)
4682                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
4683
4684         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
4685
4686         mutex_lock(&bp->hwrm_cmd_lock);
4687         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4688         if (!rc)
4689                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
4690         mutex_unlock(&bp->hwrm_cmd_lock);
4691         return rc;
4692 }
4693
4694 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
4695 {
4696         struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4697         struct hwrm_vnic_qcaps_input req = {0};
4698         int rc;
4699
4700         if (bp->hwrm_spec_code < 0x10600)
4701                 return 0;
4702
4703         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
4704         mutex_lock(&bp->hwrm_cmd_lock);
4705         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4706         if (!rc) {
4707                 u32 flags = le32_to_cpu(resp->flags);
4708
4709                 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
4710                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
4711                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
4712                 if (flags &
4713                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
4714                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
4715         }
4716         mutex_unlock(&bp->hwrm_cmd_lock);
4717         return rc;
4718 }
4719
4720 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
4721 {
4722         u16 i;
4723         u32 rc = 0;
4724
4725         if (bp->flags & BNXT_FLAG_CHIP_P5)
4726                 return 0;
4727
4728         mutex_lock(&bp->hwrm_cmd_lock);
4729         for (i = 0; i < bp->rx_nr_rings; i++) {
4730                 struct hwrm_ring_grp_alloc_input req = {0};
4731                 struct hwrm_ring_grp_alloc_output *resp =
4732                                         bp->hwrm_cmd_resp_addr;
4733                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
4734
4735                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
4736
4737                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
4738                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
4739                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
4740                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
4741
4742                 rc = _hwrm_send_message(bp, &req, sizeof(req),
4743                                         HWRM_CMD_TIMEOUT);
4744                 if (rc)
4745                         break;
4746
4747                 bp->grp_info[grp_idx].fw_grp_id =
4748                         le32_to_cpu(resp->ring_group_id);
4749         }
4750         mutex_unlock(&bp->hwrm_cmd_lock);
4751         return rc;
4752 }
4753
4754 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
4755 {
4756         u16 i;
4757         u32 rc = 0;
4758         struct hwrm_ring_grp_free_input req = {0};
4759
4760         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
4761                 return 0;
4762
4763         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
4764
4765         mutex_lock(&bp->hwrm_cmd_lock);
4766         for (i = 0; i < bp->cp_nr_rings; i++) {
4767                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
4768                         continue;
4769                 req.ring_group_id =
4770                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
4771
4772                 rc = _hwrm_send_message(bp, &req, sizeof(req),
4773                                         HWRM_CMD_TIMEOUT);
4774                 if (rc)
4775                         break;
4776                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
4777         }
4778         mutex_unlock(&bp->hwrm_cmd_lock);
4779         return rc;
4780 }
4781
4782 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
4783                                     struct bnxt_ring_struct *ring,
4784                                     u32 ring_type, u32 map_index)
4785 {
4786         int rc = 0, err = 0;
4787         struct hwrm_ring_alloc_input req = {0};
4788         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4789         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
4790         struct bnxt_ring_grp_info *grp_info;
4791         u16 ring_id;
4792
4793         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
4794
4795         req.enables = 0;
4796         if (rmem->nr_pages > 1) {
4797                 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
4798                 /* Page size is in log2 units */
4799                 req.page_size = BNXT_PAGE_SHIFT;
4800                 req.page_tbl_depth = 1;
4801         } else {
4802                 req.page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
4803         }
4804         req.fbo = 0;
4805         /* Association of ring index with doorbell index and MSIX number */
4806         req.logical_id = cpu_to_le16(map_index);
4807
4808         switch (ring_type) {
4809         case HWRM_RING_ALLOC_TX: {
4810                 struct bnxt_tx_ring_info *txr;
4811
4812                 txr = container_of(ring, struct bnxt_tx_ring_info,
4813                                    tx_ring_struct);
4814                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
4815                 /* Association of transmit ring with completion ring */
4816                 grp_info = &bp->grp_info[ring->grp_idx];
4817                 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
4818                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
4819                 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
4820                 req.queue_id = cpu_to_le16(ring->queue_id);
4821                 break;
4822         }
4823         case HWRM_RING_ALLOC_RX:
4824                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
4825                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
4826                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4827                         u16 flags = 0;
4828
4829                         /* Association of rx ring with stats context */
4830                         grp_info = &bp->grp_info[ring->grp_idx];
4831                         req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
4832                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
4833                         req.enables |= cpu_to_le32(
4834                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
4835                         if (NET_IP_ALIGN == 2)
4836                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
4837                         req.flags = cpu_to_le16(flags);
4838                 }
4839                 break;
4840         case HWRM_RING_ALLOC_AGG:
4841                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4842                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
4843                         /* Association of agg ring with rx ring */
4844                         grp_info = &bp->grp_info[ring->grp_idx];
4845                         req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
4846                         req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
4847                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
4848                         req.enables |= cpu_to_le32(
4849                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
4850                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
4851                 } else {
4852                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
4853                 }
4854                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
4855                 break;
4856         case HWRM_RING_ALLOC_CMPL:
4857                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
4858                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
4859                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4860                         /* Association of cp ring with nq */
4861                         grp_info = &bp->grp_info[map_index];
4862                         req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
4863                         req.cq_handle = cpu_to_le64(ring->handle);
4864                         req.enables |= cpu_to_le32(
4865                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
4866                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
4867                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
4868                 }
4869                 break;
4870         case HWRM_RING_ALLOC_NQ:
4871                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
4872                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
4873                 if (bp->flags & BNXT_FLAG_USING_MSIX)
4874                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
4875                 break;
4876         default:
4877                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
4878                            ring_type);
4879                 return -1;
4880         }
4881
4882         mutex_lock(&bp->hwrm_cmd_lock);
4883         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4884         err = le16_to_cpu(resp->error_code);
4885         ring_id = le16_to_cpu(resp->ring_id);
4886         mutex_unlock(&bp->hwrm_cmd_lock);
4887
4888         if (rc || err) {
4889                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
4890                            ring_type, rc, err);
4891                 return -EIO;
4892         }
4893         ring->fw_ring_id = ring_id;
4894         return rc;
4895 }
4896
4897 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
4898 {
4899         int rc;
4900
4901         if (BNXT_PF(bp)) {
4902                 struct hwrm_func_cfg_input req = {0};
4903
4904                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
4905                 req.fid = cpu_to_le16(0xffff);
4906                 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
4907                 req.async_event_cr = cpu_to_le16(idx);
4908                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4909         } else {
4910                 struct hwrm_func_vf_cfg_input req = {0};
4911
4912                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
4913                 req.enables =
4914                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
4915                 req.async_event_cr = cpu_to_le16(idx);
4916                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4917         }
4918         return rc;
4919 }
4920
4921 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
4922                         u32 map_idx, u32 xid)
4923 {
4924         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4925                 if (BNXT_PF(bp))
4926                         db->doorbell = bp->bar1 + 0x10000;
4927                 else
4928                         db->doorbell = bp->bar1 + 0x4000;
4929                 switch (ring_type) {
4930                 case HWRM_RING_ALLOC_TX:
4931                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
4932                         break;
4933                 case HWRM_RING_ALLOC_RX:
4934                 case HWRM_RING_ALLOC_AGG:
4935                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
4936                         break;
4937                 case HWRM_RING_ALLOC_CMPL:
4938                         db->db_key64 = DBR_PATH_L2;
4939                         break;
4940                 case HWRM_RING_ALLOC_NQ:
4941                         db->db_key64 = DBR_PATH_L2;
4942                         break;
4943                 }
4944                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
4945         } else {
4946                 db->doorbell = bp->bar1 + map_idx * 0x80;
4947                 switch (ring_type) {
4948                 case HWRM_RING_ALLOC_TX:
4949                         db->db_key32 = DB_KEY_TX;
4950                         break;
4951                 case HWRM_RING_ALLOC_RX:
4952                 case HWRM_RING_ALLOC_AGG:
4953                         db->db_key32 = DB_KEY_RX;
4954                         break;
4955                 case HWRM_RING_ALLOC_CMPL:
4956                         db->db_key32 = DB_KEY_CP;
4957                         break;
4958                 }
4959         }
4960 }
4961
4962 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
4963 {
4964         int i, rc = 0;
4965         u32 type;
4966
4967         if (bp->flags & BNXT_FLAG_CHIP_P5)
4968                 type = HWRM_RING_ALLOC_NQ;
4969         else
4970                 type = HWRM_RING_ALLOC_CMPL;
4971         for (i = 0; i < bp->cp_nr_rings; i++) {
4972                 struct bnxt_napi *bnapi = bp->bnapi[i];
4973                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4974                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4975                 u32 map_idx = ring->map_idx;
4976                 unsigned int vector;
4977
4978                 vector = bp->irq_tbl[map_idx].vector;
4979                 disable_irq_nosync(vector);
4980                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
4981                 if (rc) {
4982                         enable_irq(vector);
4983                         goto err_out;
4984                 }
4985                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
4986                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4987                 enable_irq(vector);
4988                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
4989
4990                 if (!i) {
4991                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
4992                         if (rc)
4993                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
4994                 }
4995         }
4996
4997         type = HWRM_RING_ALLOC_TX;
4998         for (i = 0; i < bp->tx_nr_rings; i++) {
4999                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5000                 struct bnxt_ring_struct *ring;
5001                 u32 map_idx;
5002
5003                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5004                         struct bnxt_napi *bnapi = txr->bnapi;
5005                         struct bnxt_cp_ring_info *cpr, *cpr2;
5006                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5007
5008                         cpr = &bnapi->cp_ring;
5009                         cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5010                         ring = &cpr2->cp_ring_struct;
5011                         ring->handle = BNXT_TX_HDL;
5012                         map_idx = bnapi->index;
5013                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5014                         if (rc)
5015                                 goto err_out;
5016                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5017                                     ring->fw_ring_id);
5018                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5019                 }
5020                 ring = &txr->tx_ring_struct;
5021                 map_idx = i;
5022                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5023                 if (rc)
5024                         goto err_out;
5025                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5026         }
5027
5028         type = HWRM_RING_ALLOC_RX;
5029         for (i = 0; i < bp->rx_nr_rings; i++) {
5030                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5031                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5032                 struct bnxt_napi *bnapi = rxr->bnapi;
5033                 u32 map_idx = bnapi->index;
5034
5035                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5036                 if (rc)
5037                         goto err_out;
5038                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5039                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5040                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5041                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5042                         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5043                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5044                         struct bnxt_cp_ring_info *cpr2;
5045
5046                         cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5047                         ring = &cpr2->cp_ring_struct;
5048                         ring->handle = BNXT_RX_HDL;
5049                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5050                         if (rc)
5051                                 goto err_out;
5052                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5053                                     ring->fw_ring_id);
5054                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5055                 }
5056         }
5057
5058         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
5059                 type = HWRM_RING_ALLOC_AGG;
5060                 for (i = 0; i < bp->rx_nr_rings; i++) {
5061                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5062                         struct bnxt_ring_struct *ring =
5063                                                 &rxr->rx_agg_ring_struct;
5064                         u32 grp_idx = ring->grp_idx;
5065                         u32 map_idx = grp_idx + bp->rx_nr_rings;
5066
5067                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5068                         if (rc)
5069                                 goto err_out;
5070
5071                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5072                                     ring->fw_ring_id);
5073                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5074                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5075                 }
5076         }
5077 err_out:
5078         return rc;
5079 }
5080
5081 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5082                                    struct bnxt_ring_struct *ring,
5083                                    u32 ring_type, int cmpl_ring_id)
5084 {
5085         int rc;
5086         struct hwrm_ring_free_input req = {0};
5087         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5088         u16 error_code;
5089
5090         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5091         req.ring_type = ring_type;
5092         req.ring_id = cpu_to_le16(ring->fw_ring_id);
5093
5094         mutex_lock(&bp->hwrm_cmd_lock);
5095         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5096         error_code = le16_to_cpu(resp->error_code);
5097         mutex_unlock(&bp->hwrm_cmd_lock);
5098
5099         if (rc || error_code) {
5100                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5101                            ring_type, rc, error_code);
5102                 return -EIO;
5103         }
5104         return 0;
5105 }
5106
5107 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5108 {
5109         u32 type;
5110         int i;
5111
5112         if (!bp->bnapi)
5113                 return;
5114
5115         for (i = 0; i < bp->tx_nr_rings; i++) {
5116                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5117                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5118                 u32 cmpl_ring_id;
5119
5120                 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5121                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5122                         hwrm_ring_free_send_msg(bp, ring,
5123                                                 RING_FREE_REQ_RING_TYPE_TX,
5124                                                 close_path ? cmpl_ring_id :
5125                                                 INVALID_HW_RING_ID);
5126                         ring->fw_ring_id = INVALID_HW_RING_ID;
5127                 }
5128         }
5129
5130         for (i = 0; i < bp->rx_nr_rings; i++) {
5131                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5132                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5133                 u32 grp_idx = rxr->bnapi->index;
5134                 u32 cmpl_ring_id;
5135
5136                 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5137                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5138                         hwrm_ring_free_send_msg(bp, ring,
5139                                                 RING_FREE_REQ_RING_TYPE_RX,
5140                                                 close_path ? cmpl_ring_id :
5141                                                 INVALID_HW_RING_ID);
5142                         ring->fw_ring_id = INVALID_HW_RING_ID;
5143                         bp->grp_info[grp_idx].rx_fw_ring_id =
5144                                 INVALID_HW_RING_ID;
5145                 }
5146         }
5147
5148         if (bp->flags & BNXT_FLAG_CHIP_P5)
5149                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5150         else
5151                 type = RING_FREE_REQ_RING_TYPE_RX;
5152         for (i = 0; i < bp->rx_nr_rings; i++) {
5153                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5154                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5155                 u32 grp_idx = rxr->bnapi->index;
5156                 u32 cmpl_ring_id;
5157
5158                 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5159                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5160                         hwrm_ring_free_send_msg(bp, ring, type,
5161                                                 close_path ? cmpl_ring_id :
5162                                                 INVALID_HW_RING_ID);
5163                         ring->fw_ring_id = INVALID_HW_RING_ID;
5164                         bp->grp_info[grp_idx].agg_fw_ring_id =
5165                                 INVALID_HW_RING_ID;
5166                 }
5167         }
5168
5169         /* The completion rings are about to be freed.  After that the
5170          * IRQ doorbell will not work anymore.  So we need to disable
5171          * IRQ here.
5172          */
5173         bnxt_disable_int_sync(bp);
5174
5175         if (bp->flags & BNXT_FLAG_CHIP_P5)
5176                 type = RING_FREE_REQ_RING_TYPE_NQ;
5177         else
5178                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5179         for (i = 0; i < bp->cp_nr_rings; i++) {
5180                 struct bnxt_napi *bnapi = bp->bnapi[i];
5181                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5182                 struct bnxt_ring_struct *ring;
5183                 int j;
5184
5185                 for (j = 0; j < 2; j++) {
5186                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5187
5188                         if (cpr2) {
5189                                 ring = &cpr2->cp_ring_struct;
5190                                 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5191                                         continue;
5192                                 hwrm_ring_free_send_msg(bp, ring,
5193                                         RING_FREE_REQ_RING_TYPE_L2_CMPL,
5194                                         INVALID_HW_RING_ID);
5195                                 ring->fw_ring_id = INVALID_HW_RING_ID;
5196                         }
5197                 }
5198                 ring = &cpr->cp_ring_struct;
5199                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5200                         hwrm_ring_free_send_msg(bp, ring, type,
5201                                                 INVALID_HW_RING_ID);
5202                         ring->fw_ring_id = INVALID_HW_RING_ID;
5203                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5204                 }
5205         }
5206 }
5207
5208 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5209                            bool shared);
5210
5211 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5212 {
5213         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5214         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5215         struct hwrm_func_qcfg_input req = {0};
5216         int rc;
5217
5218         if (bp->hwrm_spec_code < 0x10601)
5219                 return 0;
5220
5221         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5222         req.fid = cpu_to_le16(0xffff);
5223         mutex_lock(&bp->hwrm_cmd_lock);
5224         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5225         if (rc) {
5226                 mutex_unlock(&bp->hwrm_cmd_lock);
5227                 return -EIO;
5228         }
5229
5230         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5231         if (BNXT_NEW_RM(bp)) {
5232                 u16 cp, stats;
5233
5234                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5235                 hw_resc->resv_hw_ring_grps =
5236                         le32_to_cpu(resp->alloc_hw_ring_grps);
5237                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5238                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
5239                 stats = le16_to_cpu(resp->alloc_stat_ctx);
5240                 hw_resc->resv_irqs = cp;
5241                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5242                         int rx = hw_resc->resv_rx_rings;
5243                         int tx = hw_resc->resv_tx_rings;
5244
5245                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5246                                 rx >>= 1;
5247                         if (cp < (rx + tx)) {
5248                                 bnxt_trim_rings(bp, &rx, &tx, cp, false);
5249                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5250                                         rx <<= 1;
5251                                 hw_resc->resv_rx_rings = rx;
5252                                 hw_resc->resv_tx_rings = tx;
5253                         }
5254                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
5255                         hw_resc->resv_hw_ring_grps = rx;
5256                 }
5257                 hw_resc->resv_cp_rings = cp;
5258                 hw_resc->resv_stat_ctxs = stats;
5259         }
5260         mutex_unlock(&bp->hwrm_cmd_lock);
5261         return 0;
5262 }
5263
5264 /* Caller must hold bp->hwrm_cmd_lock */
5265 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5266 {
5267         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5268         struct hwrm_func_qcfg_input req = {0};
5269         int rc;
5270
5271         if (bp->hwrm_spec_code < 0x10601)
5272                 return 0;
5273
5274         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5275         req.fid = cpu_to_le16(fid);
5276         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5277         if (!rc)
5278                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5279
5280         return rc;
5281 }
5282
5283 static bool bnxt_rfs_supported(struct bnxt *bp);
5284
5285 static void
5286 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5287                              int tx_rings, int rx_rings, int ring_grps,
5288                              int cp_rings, int stats, int vnics)
5289 {
5290         u32 enables = 0;
5291
5292         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5293         req->fid = cpu_to_le16(0xffff);
5294         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5295         req->num_tx_rings = cpu_to_le16(tx_rings);
5296         if (BNXT_NEW_RM(bp)) {
5297                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
5298                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5299                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5300                         enables |= tx_rings + ring_grps ?
5301                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS |
5302                                    FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5303                         enables |= rx_rings ?
5304                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5305                 } else {
5306                         enables |= cp_rings ?
5307                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS |
5308                                    FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5309                         enables |= ring_grps ?
5310                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
5311                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5312                 }
5313                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
5314
5315                 req->num_rx_rings = cpu_to_le16(rx_rings);
5316                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5317                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5318                         req->num_msix = cpu_to_le16(cp_rings);
5319                         req->num_rsscos_ctxs =
5320                                 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5321                 } else {
5322                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
5323                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5324                         req->num_rsscos_ctxs = cpu_to_le16(1);
5325                         if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
5326                             bnxt_rfs_supported(bp))
5327                                 req->num_rsscos_ctxs =
5328                                         cpu_to_le16(ring_grps + 1);
5329                 }
5330                 req->num_stat_ctxs = cpu_to_le16(stats);
5331                 req->num_vnics = cpu_to_le16(vnics);
5332         }
5333         req->enables = cpu_to_le32(enables);
5334 }
5335
5336 static void
5337 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
5338                              struct hwrm_func_vf_cfg_input *req, int tx_rings,
5339                              int rx_rings, int ring_grps, int cp_rings,
5340                              int stats, int vnics)
5341 {
5342         u32 enables = 0;
5343
5344         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
5345         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5346         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
5347                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5348         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5349                 enables |= tx_rings + ring_grps ?
5350                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS |
5351                            FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5352         } else {
5353                 enables |= cp_rings ?
5354                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS |
5355                            FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5356                 enables |= ring_grps ?
5357                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
5358         }
5359         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
5360         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
5361
5362         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
5363         req->num_tx_rings = cpu_to_le16(tx_rings);
5364         req->num_rx_rings = cpu_to_le16(rx_rings);
5365         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5366                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5367                 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5368         } else {
5369                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
5370                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5371                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
5372         }
5373         req->num_stat_ctxs = cpu_to_le16(stats);
5374         req->num_vnics = cpu_to_le16(vnics);
5375
5376         req->enables = cpu_to_le32(enables);
5377 }
5378
5379 static int
5380 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5381                            int ring_grps, int cp_rings, int stats, int vnics)
5382 {
5383         struct hwrm_func_cfg_input req = {0};
5384         int rc;
5385
5386         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5387                                      cp_rings, stats, vnics);
5388         if (!req.enables)
5389                 return 0;
5390
5391         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5392         if (rc)
5393                 return -ENOMEM;
5394
5395         if (bp->hwrm_spec_code < 0x10601)
5396                 bp->hw_resc.resv_tx_rings = tx_rings;
5397
5398         rc = bnxt_hwrm_get_rings(bp);
5399         return rc;
5400 }
5401
5402 static int
5403 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5404                            int ring_grps, int cp_rings, int stats, int vnics)
5405 {
5406         struct hwrm_func_vf_cfg_input req = {0};
5407         int rc;
5408
5409         if (!BNXT_NEW_RM(bp)) {
5410                 bp->hw_resc.resv_tx_rings = tx_rings;
5411                 return 0;
5412         }
5413
5414         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5415                                      cp_rings, stats, vnics);
5416         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5417         if (rc)
5418                 return -ENOMEM;
5419
5420         rc = bnxt_hwrm_get_rings(bp);
5421         return rc;
5422 }
5423
5424 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
5425                                    int cp, int stat, int vnic)
5426 {
5427         if (BNXT_PF(bp))
5428                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
5429                                                   vnic);
5430         else
5431                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
5432                                                   vnic);
5433 }
5434
5435 int bnxt_nq_rings_in_use(struct bnxt *bp)
5436 {
5437         int cp = bp->cp_nr_rings;
5438         int ulp_msix, ulp_base;
5439
5440         ulp_msix = bnxt_get_ulp_msix_num(bp);
5441         if (ulp_msix) {
5442                 ulp_base = bnxt_get_ulp_msix_base(bp);
5443                 cp += ulp_msix;
5444                 if ((ulp_base + ulp_msix) > cp)
5445                         cp = ulp_base + ulp_msix;
5446         }
5447         return cp;
5448 }
5449
5450 static int bnxt_cp_rings_in_use(struct bnxt *bp)
5451 {
5452         int cp;
5453
5454         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5455                 return bnxt_nq_rings_in_use(bp);
5456
5457         cp = bp->tx_nr_rings + bp->rx_nr_rings;
5458         return cp;
5459 }
5460
5461 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
5462 {
5463         return bp->cp_nr_rings + bnxt_get_ulp_stat_ctxs(bp);
5464 }
5465
5466 static bool bnxt_need_reserve_rings(struct bnxt *bp)
5467 {
5468         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5469         int cp = bnxt_cp_rings_in_use(bp);
5470         int nq = bnxt_nq_rings_in_use(bp);
5471         int rx = bp->rx_nr_rings, stat;
5472         int vnic = 1, grp = rx;
5473
5474         if (bp->hwrm_spec_code < 0x10601)
5475                 return false;
5476
5477         if (hw_resc->resv_tx_rings != bp->tx_nr_rings)
5478                 return true;
5479
5480         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
5481                 vnic = rx + 1;
5482         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5483                 rx <<= 1;
5484         stat = bnxt_get_func_stat_ctxs(bp);
5485         if (BNXT_NEW_RM(bp) &&
5486             (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
5487              hw_resc->resv_irqs < nq || hw_resc->resv_vnics != vnic ||
5488              hw_resc->resv_stat_ctxs != stat ||
5489              (hw_resc->resv_hw_ring_grps != grp &&
5490               !(bp->flags & BNXT_FLAG_CHIP_P5))))
5491                 return true;
5492         return false;
5493 }
5494
5495 static int __bnxt_reserve_rings(struct bnxt *bp)
5496 {
5497         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5498         int cp = bnxt_nq_rings_in_use(bp);
5499         int tx = bp->tx_nr_rings;
5500         int rx = bp->rx_nr_rings;
5501         int grp, rx_rings, rc;
5502         int vnic = 1, stat;
5503         bool sh = false;
5504
5505         if (!bnxt_need_reserve_rings(bp))
5506                 return 0;
5507
5508         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5509                 sh = true;
5510         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
5511                 vnic = rx + 1;
5512         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5513                 rx <<= 1;
5514         grp = bp->rx_nr_rings;
5515         stat = bnxt_get_func_stat_ctxs(bp);
5516
5517         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
5518         if (rc)
5519                 return rc;
5520
5521         tx = hw_resc->resv_tx_rings;
5522         if (BNXT_NEW_RM(bp)) {
5523                 rx = hw_resc->resv_rx_rings;
5524                 cp = hw_resc->resv_irqs;
5525                 grp = hw_resc->resv_hw_ring_grps;
5526                 vnic = hw_resc->resv_vnics;
5527                 stat = hw_resc->resv_stat_ctxs;
5528         }
5529
5530         rx_rings = rx;
5531         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
5532                 if (rx >= 2) {
5533                         rx_rings = rx >> 1;
5534                 } else {
5535                         if (netif_running(bp->dev))
5536                                 return -ENOMEM;
5537
5538                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
5539                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
5540                         bp->dev->hw_features &= ~NETIF_F_LRO;
5541                         bp->dev->features &= ~NETIF_F_LRO;
5542                         bnxt_set_ring_params(bp);
5543                 }
5544         }
5545         rx_rings = min_t(int, rx_rings, grp);
5546         cp = min_t(int, cp, bp->cp_nr_rings);
5547         if (stat > bnxt_get_ulp_stat_ctxs(bp))
5548                 stat -= bnxt_get_ulp_stat_ctxs(bp);
5549         cp = min_t(int, cp, stat);
5550         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
5551         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5552                 rx = rx_rings << 1;
5553         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
5554         bp->tx_nr_rings = tx;
5555         bp->rx_nr_rings = rx_rings;
5556         bp->cp_nr_rings = cp;
5557
5558         if (!tx || !rx || !cp || !grp || !vnic || !stat)
5559                 return -ENOMEM;
5560
5561         return rc;
5562 }
5563
5564 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5565                                     int ring_grps, int cp_rings, int stats,
5566                                     int vnics)
5567 {
5568         struct hwrm_func_vf_cfg_input req = {0};
5569         u32 flags;
5570         int rc;
5571
5572         if (!BNXT_NEW_RM(bp))
5573                 return 0;
5574
5575         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5576                                      cp_rings, stats, vnics);
5577         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
5578                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
5579                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
5580                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
5581                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
5582                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
5583         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5584                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
5585
5586         req.flags = cpu_to_le32(flags);
5587         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5588         if (rc)
5589                 return -ENOMEM;
5590         return 0;
5591 }
5592
5593 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5594                                     int ring_grps, int cp_rings, int stats,
5595                                     int vnics)
5596 {
5597         struct hwrm_func_cfg_input req = {0};
5598         u32 flags;
5599         int rc;
5600
5601         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5602                                      cp_rings, stats, vnics);
5603         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
5604         if (BNXT_NEW_RM(bp)) {
5605                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
5606                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
5607                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
5608                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
5609                 if (bp->flags & BNXT_FLAG_CHIP_P5)
5610                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
5611                                  FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
5612                 else
5613                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
5614         }
5615
5616         req.flags = cpu_to_le32(flags);
5617         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5618         if (rc)
5619                 return -ENOMEM;
5620         return 0;
5621 }
5622
5623 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5624                                  int ring_grps, int cp_rings, int stats,
5625                                  int vnics)
5626 {
5627         if (bp->hwrm_spec_code < 0x10801)
5628                 return 0;
5629
5630         if (BNXT_PF(bp))
5631                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
5632                                                 ring_grps, cp_rings, stats,
5633                                                 vnics);
5634
5635         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
5636                                         cp_rings, stats, vnics);
5637 }
5638
5639 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
5640 {
5641         struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5642         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5643         struct hwrm_ring_aggint_qcaps_input req = {0};
5644         int rc;
5645
5646         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
5647         coal_cap->num_cmpl_dma_aggr_max = 63;
5648         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
5649         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
5650         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
5651         coal_cap->int_lat_tmr_min_max = 65535;
5652         coal_cap->int_lat_tmr_max_max = 65535;
5653         coal_cap->num_cmpl_aggr_int_max = 65535;
5654         coal_cap->timer_units = 80;
5655
5656         if (bp->hwrm_spec_code < 0x10902)
5657                 return;
5658
5659         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
5660         mutex_lock(&bp->hwrm_cmd_lock);
5661         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5662         if (!rc) {
5663                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
5664                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
5665                 coal_cap->num_cmpl_dma_aggr_max =
5666                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
5667                 coal_cap->num_cmpl_dma_aggr_during_int_max =
5668                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
5669                 coal_cap->cmpl_aggr_dma_tmr_max =
5670                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
5671                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
5672                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
5673                 coal_cap->int_lat_tmr_min_max =
5674                         le16_to_cpu(resp->int_lat_tmr_min_max);
5675                 coal_cap->int_lat_tmr_max_max =
5676                         le16_to_cpu(resp->int_lat_tmr_max_max);
5677                 coal_cap->num_cmpl_aggr_int_max =
5678                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
5679                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
5680         }
5681         mutex_unlock(&bp->hwrm_cmd_lock);
5682 }
5683
5684 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
5685 {
5686         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5687
5688         return usec * 1000 / coal_cap->timer_units;
5689 }
5690
5691 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
5692         struct bnxt_coal *hw_coal,
5693         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
5694 {
5695         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5696         u32 cmpl_params = coal_cap->cmpl_params;
5697         u16 val, tmr, max, flags = 0;
5698
5699         max = hw_coal->bufs_per_record * 128;
5700         if (hw_coal->budget)
5701                 max = hw_coal->bufs_per_record * hw_coal->budget;
5702         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
5703
5704         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
5705         req->num_cmpl_aggr_int = cpu_to_le16(val);
5706
5707         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
5708         req->num_cmpl_dma_aggr = cpu_to_le16(val);
5709
5710         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
5711                       coal_cap->num_cmpl_dma_aggr_during_int_max);
5712         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
5713
5714         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
5715         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
5716         req->int_lat_tmr_max = cpu_to_le16(tmr);
5717
5718         /* min timer set to 1/2 of interrupt timer */
5719         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
5720                 val = tmr / 2;
5721                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
5722                 req->int_lat_tmr_min = cpu_to_le16(val);
5723                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
5724         }
5725
5726         /* buf timer set to 1/4 of interrupt timer */
5727         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
5728         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
5729
5730         if (cmpl_params &
5731             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
5732                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
5733                 val = clamp_t(u16, tmr, 1,
5734                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
5735                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(tmr);
5736                 req->enables |=
5737                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
5738         }
5739
5740         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
5741                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
5742         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
5743             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
5744                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
5745         req->flags = cpu_to_le16(flags);
5746         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
5747 }
5748
5749 /* Caller holds bp->hwrm_cmd_lock */
5750 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
5751                                    struct bnxt_coal *hw_coal)
5752 {
5753         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
5754         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5755         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5756         u32 nq_params = coal_cap->nq_params;
5757         u16 tmr;
5758
5759         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
5760                 return 0;
5761
5762         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
5763                                -1, -1);
5764         req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
5765         req.flags =
5766                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
5767
5768         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
5769         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
5770         req.int_lat_tmr_min = cpu_to_le16(tmr);
5771         req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
5772         return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5773 }
5774
5775 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
5776 {
5777         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
5778         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5779         struct bnxt_coal coal;
5780
5781         /* Tick values in micro seconds.
5782          * 1 coal_buf x bufs_per_record = 1 completion record.
5783          */
5784         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
5785
5786         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
5787         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
5788
5789         if (!bnapi->rx_ring)
5790                 return -ENODEV;
5791
5792         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
5793                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
5794
5795         bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
5796
5797         req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
5798
5799         return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
5800                                  HWRM_CMD_TIMEOUT);
5801 }
5802
5803 int bnxt_hwrm_set_coal(struct bnxt *bp)
5804 {
5805         int i, rc = 0;
5806         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
5807                                                            req_tx = {0}, *req;
5808
5809         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
5810                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
5811         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
5812                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
5813
5814         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
5815         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
5816
5817         mutex_lock(&bp->hwrm_cmd_lock);
5818         for (i = 0; i < bp->cp_nr_rings; i++) {
5819                 struct bnxt_napi *bnapi = bp->bnapi[i];
5820                 struct bnxt_coal *hw_coal;
5821                 u16 ring_id;
5822
5823                 req = &req_rx;
5824                 if (!bnapi->rx_ring) {
5825                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
5826                         req = &req_tx;
5827                 } else {
5828                         ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
5829                 }
5830                 req->ring_id = cpu_to_le16(ring_id);
5831
5832                 rc = _hwrm_send_message(bp, req, sizeof(*req),
5833                                         HWRM_CMD_TIMEOUT);
5834                 if (rc)
5835                         break;
5836
5837                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5838                         continue;
5839
5840                 if (bnapi->rx_ring && bnapi->tx_ring) {
5841                         req = &req_tx;
5842                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
5843                         req->ring_id = cpu_to_le16(ring_id);
5844                         rc = _hwrm_send_message(bp, req, sizeof(*req),
5845                                                 HWRM_CMD_TIMEOUT);
5846                         if (rc)
5847                                 break;
5848                 }
5849                 if (bnapi->rx_ring)
5850                         hw_coal = &bp->rx_coal;
5851                 else
5852                         hw_coal = &bp->tx_coal;
5853                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
5854         }
5855         mutex_unlock(&bp->hwrm_cmd_lock);
5856         return rc;
5857 }
5858
5859 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
5860 {
5861         int rc = 0, i;
5862         struct hwrm_stat_ctx_free_input req = {0};
5863
5864         if (!bp->bnapi)
5865                 return 0;
5866
5867         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5868                 return 0;
5869
5870         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
5871
5872         mutex_lock(&bp->hwrm_cmd_lock);
5873         for (i = 0; i < bp->cp_nr_rings; i++) {
5874                 struct bnxt_napi *bnapi = bp->bnapi[i];
5875                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5876
5877                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
5878                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
5879
5880                         rc = _hwrm_send_message(bp, &req, sizeof(req),
5881                                                 HWRM_CMD_TIMEOUT);
5882                         if (rc)
5883                                 break;
5884
5885                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
5886                 }
5887         }
5888         mutex_unlock(&bp->hwrm_cmd_lock);
5889         return rc;
5890 }
5891
5892 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
5893 {
5894         int rc = 0, i;
5895         struct hwrm_stat_ctx_alloc_input req = {0};
5896         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5897
5898         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5899                 return 0;
5900
5901         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
5902
5903         req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
5904
5905         mutex_lock(&bp->hwrm_cmd_lock);
5906         for (i = 0; i < bp->cp_nr_rings; i++) {
5907                 struct bnxt_napi *bnapi = bp->bnapi[i];
5908                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5909
5910                 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
5911
5912                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5913                                         HWRM_CMD_TIMEOUT);
5914                 if (rc)
5915                         break;
5916
5917                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
5918
5919                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
5920         }
5921         mutex_unlock(&bp->hwrm_cmd_lock);
5922         return rc;
5923 }
5924
5925 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
5926 {
5927         struct hwrm_func_qcfg_input req = {0};
5928         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5929         u16 flags;
5930         int rc;
5931
5932         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5933         req.fid = cpu_to_le16(0xffff);
5934         mutex_lock(&bp->hwrm_cmd_lock);
5935         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5936         if (rc)
5937                 goto func_qcfg_exit;
5938
5939 #ifdef CONFIG_BNXT_SRIOV
5940         if (BNXT_VF(bp)) {
5941                 struct bnxt_vf_info *vf = &bp->vf;
5942
5943                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
5944         }
5945 #endif
5946         flags = le16_to_cpu(resp->flags);
5947         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
5948                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
5949                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
5950                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
5951                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
5952         }
5953         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
5954                 bp->flags |= BNXT_FLAG_MULTI_HOST;
5955
5956         switch (resp->port_partition_type) {
5957         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
5958         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
5959         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
5960                 bp->port_partition_type = resp->port_partition_type;
5961                 break;
5962         }
5963         if (bp->hwrm_spec_code < 0x10707 ||
5964             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
5965                 bp->br_mode = BRIDGE_MODE_VEB;
5966         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
5967                 bp->br_mode = BRIDGE_MODE_VEPA;
5968         else
5969                 bp->br_mode = BRIDGE_MODE_UNDEF;
5970
5971         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
5972         if (!bp->max_mtu)
5973                 bp->max_mtu = BNXT_MAX_MTU;
5974
5975 func_qcfg_exit:
5976         mutex_unlock(&bp->hwrm_cmd_lock);
5977         return rc;
5978 }
5979
5980 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
5981 {
5982         struct hwrm_func_backing_store_qcaps_input req = {0};
5983         struct hwrm_func_backing_store_qcaps_output *resp =
5984                 bp->hwrm_cmd_resp_addr;
5985         int rc;
5986
5987         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
5988                 return 0;
5989
5990         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
5991         mutex_lock(&bp->hwrm_cmd_lock);
5992         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5993         if (!rc) {
5994                 struct bnxt_ctx_pg_info *ctx_pg;
5995                 struct bnxt_ctx_mem_info *ctx;
5996                 int i;
5997
5998                 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
5999                 if (!ctx) {
6000                         rc = -ENOMEM;
6001                         goto ctx_err;
6002                 }
6003                 ctx_pg = kzalloc(sizeof(*ctx_pg) * (bp->max_q + 1), GFP_KERNEL);
6004                 if (!ctx_pg) {
6005                         kfree(ctx);
6006                         rc = -ENOMEM;
6007                         goto ctx_err;
6008                 }
6009                 for (i = 0; i < bp->max_q + 1; i++, ctx_pg++)
6010                         ctx->tqm_mem[i] = ctx_pg;
6011
6012                 bp->ctx = ctx;
6013                 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6014                 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6015                 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6016                 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6017                 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6018                 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6019                 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6020                 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6021                 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6022                 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6023                 ctx->vnic_max_vnic_entries =
6024                         le16_to_cpu(resp->vnic_max_vnic_entries);
6025                 ctx->vnic_max_ring_table_entries =
6026                         le16_to_cpu(resp->vnic_max_ring_table_entries);
6027                 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6028                 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6029                 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6030                 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6031                 ctx->tqm_min_entries_per_ring =
6032                         le32_to_cpu(resp->tqm_min_entries_per_ring);
6033                 ctx->tqm_max_entries_per_ring =
6034                         le32_to_cpu(resp->tqm_max_entries_per_ring);
6035                 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6036                 if (!ctx->tqm_entries_multiple)
6037                         ctx->tqm_entries_multiple = 1;
6038                 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6039                 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
6040                 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6041                 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6042         } else {
6043                 rc = 0;
6044         }
6045 ctx_err:
6046         mutex_unlock(&bp->hwrm_cmd_lock);
6047         return rc;
6048 }
6049
6050 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6051                                   __le64 *pg_dir)
6052 {
6053         u8 pg_size = 0;
6054
6055         if (BNXT_PAGE_SHIFT == 13)
6056                 pg_size = 1 << 4;
6057         else if (BNXT_PAGE_SIZE == 16)
6058                 pg_size = 2 << 4;
6059
6060         *pg_attr = pg_size;
6061         if (rmem->depth >= 1) {
6062                 if (rmem->depth == 2)
6063                         *pg_attr |= 2;
6064                 else
6065                         *pg_attr |= 1;
6066                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6067         } else {
6068                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6069         }
6070 }
6071
6072 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
6073         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
6074          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
6075          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
6076          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
6077          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6078
6079 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6080 {
6081         struct hwrm_func_backing_store_cfg_input req = {0};
6082         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6083         struct bnxt_ctx_pg_info *ctx_pg;
6084         __le32 *num_entries;
6085         __le64 *pg_dir;
6086         u8 *pg_attr;
6087         int i, rc;
6088         u32 ena;
6089
6090         if (!ctx)
6091                 return 0;
6092
6093         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6094         req.enables = cpu_to_le32(enables);
6095
6096         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6097                 ctx_pg = &ctx->qp_mem;
6098                 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6099                 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6100                 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6101                 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6102                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6103                                       &req.qpc_pg_size_qpc_lvl,
6104                                       &req.qpc_page_dir);
6105         }
6106         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6107                 ctx_pg = &ctx->srq_mem;
6108                 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6109                 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6110                 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6111                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6112                                       &req.srq_pg_size_srq_lvl,
6113                                       &req.srq_page_dir);
6114         }
6115         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6116                 ctx_pg = &ctx->cq_mem;
6117                 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6118                 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6119                 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6120                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6121                                       &req.cq_page_dir);
6122         }
6123         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6124                 ctx_pg = &ctx->vnic_mem;
6125                 req.vnic_num_vnic_entries =
6126                         cpu_to_le16(ctx->vnic_max_vnic_entries);
6127                 req.vnic_num_ring_table_entries =
6128                         cpu_to_le16(ctx->vnic_max_ring_table_entries);
6129                 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6130                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6131                                       &req.vnic_pg_size_vnic_lvl,
6132                                       &req.vnic_page_dir);
6133         }
6134         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6135                 ctx_pg = &ctx->stat_mem;
6136                 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6137                 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6138                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6139                                       &req.stat_pg_size_stat_lvl,
6140                                       &req.stat_page_dir);
6141         }
6142         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
6143                 ctx_pg = &ctx->mrav_mem;
6144                 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
6145                 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
6146                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6147                                       &req.mrav_pg_size_mrav_lvl,
6148                                       &req.mrav_page_dir);
6149         }
6150         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
6151                 ctx_pg = &ctx->tim_mem;
6152                 req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
6153                 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
6154                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6155                                       &req.tim_pg_size_tim_lvl,
6156                                       &req.tim_page_dir);
6157         }
6158         for (i = 0, num_entries = &req.tqm_sp_num_entries,
6159              pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6160              pg_dir = &req.tqm_sp_page_dir,
6161              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6162              i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6163                 if (!(enables & ena))
6164                         continue;
6165
6166                 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6167                 ctx_pg = ctx->tqm_mem[i];
6168                 *num_entries = cpu_to_le32(ctx_pg->entries);
6169                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6170         }
6171         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6172         if (rc)
6173                 rc = -EIO;
6174         return rc;
6175 }
6176
6177 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
6178                                   struct bnxt_ctx_pg_info *ctx_pg)
6179 {
6180         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6181
6182         rmem->page_size = BNXT_PAGE_SIZE;
6183         rmem->pg_arr = ctx_pg->ctx_pg_arr;
6184         rmem->dma_arr = ctx_pg->ctx_dma_arr;
6185         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
6186         if (rmem->depth >= 1)
6187                 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
6188         return bnxt_alloc_ring(bp, rmem);
6189 }
6190
6191 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
6192                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
6193                                   u8 depth)
6194 {
6195         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6196         int rc;
6197
6198         if (!mem_size)
6199                 return 0;
6200
6201         ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6202         if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
6203                 ctx_pg->nr_pages = 0;
6204                 return -EINVAL;
6205         }
6206         if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
6207                 int nr_tbls, i;
6208
6209                 rmem->depth = 2;
6210                 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
6211                                              GFP_KERNEL);
6212                 if (!ctx_pg->ctx_pg_tbl)
6213                         return -ENOMEM;
6214                 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
6215                 rmem->nr_pages = nr_tbls;
6216                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6217                 if (rc)
6218                         return rc;
6219                 for (i = 0; i < nr_tbls; i++) {
6220                         struct bnxt_ctx_pg_info *pg_tbl;
6221
6222                         pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
6223                         if (!pg_tbl)
6224                                 return -ENOMEM;
6225                         ctx_pg->ctx_pg_tbl[i] = pg_tbl;
6226                         rmem = &pg_tbl->ring_mem;
6227                         rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
6228                         rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
6229                         rmem->depth = 1;
6230                         rmem->nr_pages = MAX_CTX_PAGES;
6231                         if (i == (nr_tbls - 1)) {
6232                                 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
6233
6234                                 if (rem)
6235                                         rmem->nr_pages = rem;
6236                         }
6237                         rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
6238                         if (rc)
6239                                 break;
6240                 }
6241         } else {
6242                 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6243                 if (rmem->nr_pages > 1 || depth)
6244                         rmem->depth = 1;
6245                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6246         }
6247         return rc;
6248 }
6249
6250 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
6251                                   struct bnxt_ctx_pg_info *ctx_pg)
6252 {
6253         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6254
6255         if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
6256             ctx_pg->ctx_pg_tbl) {
6257                 int i, nr_tbls = rmem->nr_pages;
6258
6259                 for (i = 0; i < nr_tbls; i++) {
6260                         struct bnxt_ctx_pg_info *pg_tbl;
6261                         struct bnxt_ring_mem_info *rmem2;
6262
6263                         pg_tbl = ctx_pg->ctx_pg_tbl[i];
6264                         if (!pg_tbl)
6265                                 continue;
6266                         rmem2 = &pg_tbl->ring_mem;
6267                         bnxt_free_ring(bp, rmem2);
6268                         ctx_pg->ctx_pg_arr[i] = NULL;
6269                         kfree(pg_tbl);
6270                         ctx_pg->ctx_pg_tbl[i] = NULL;
6271                 }
6272                 kfree(ctx_pg->ctx_pg_tbl);
6273                 ctx_pg->ctx_pg_tbl = NULL;
6274         }
6275         bnxt_free_ring(bp, rmem);
6276         ctx_pg->nr_pages = 0;
6277 }
6278
6279 static void bnxt_free_ctx_mem(struct bnxt *bp)
6280 {
6281         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6282         int i;
6283
6284         if (!ctx)
6285                 return;
6286
6287         if (ctx->tqm_mem[0]) {
6288                 for (i = 0; i < bp->max_q + 1; i++)
6289                         bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
6290                 kfree(ctx->tqm_mem[0]);
6291                 ctx->tqm_mem[0] = NULL;
6292         }
6293
6294         bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
6295         bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
6296         bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
6297         bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
6298         bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
6299         bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
6300         bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
6301         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
6302 }
6303
6304 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
6305 {
6306         struct bnxt_ctx_pg_info *ctx_pg;
6307         struct bnxt_ctx_mem_info *ctx;
6308         u32 mem_size, ena, entries;
6309         u32 extra_srqs = 0;
6310         u32 extra_qps = 0;
6311         u8 pg_lvl = 1;
6312         int i, rc;
6313
6314         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
6315         if (rc) {
6316                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
6317                            rc);
6318                 return rc;
6319         }
6320         ctx = bp->ctx;
6321         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
6322                 return 0;
6323
6324         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
6325                 pg_lvl = 2;
6326                 extra_qps = 65536;
6327                 extra_srqs = 8192;
6328         }
6329
6330         ctx_pg = &ctx->qp_mem;
6331         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
6332                           extra_qps;
6333         mem_size = ctx->qp_entry_size * ctx_pg->entries;
6334         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
6335         if (rc)
6336                 return rc;
6337
6338         ctx_pg = &ctx->srq_mem;
6339         ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
6340         mem_size = ctx->srq_entry_size * ctx_pg->entries;
6341         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
6342         if (rc)
6343                 return rc;
6344
6345         ctx_pg = &ctx->cq_mem;
6346         ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
6347         mem_size = ctx->cq_entry_size * ctx_pg->entries;
6348         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
6349         if (rc)
6350                 return rc;
6351
6352         ctx_pg = &ctx->vnic_mem;
6353         ctx_pg->entries = ctx->vnic_max_vnic_entries +
6354                           ctx->vnic_max_ring_table_entries;
6355         mem_size = ctx->vnic_entry_size * ctx_pg->entries;
6356         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6357         if (rc)
6358                 return rc;
6359
6360         ctx_pg = &ctx->stat_mem;
6361         ctx_pg->entries = ctx->stat_max_entries;
6362         mem_size = ctx->stat_entry_size * ctx_pg->entries;
6363         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6364         if (rc)
6365                 return rc;
6366
6367         ena = 0;
6368         if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
6369                 goto skip_rdma;
6370
6371         ctx_pg = &ctx->mrav_mem;
6372         ctx_pg->entries = extra_qps * 4;
6373         mem_size = ctx->mrav_entry_size * ctx_pg->entries;
6374         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2);
6375         if (rc)
6376                 return rc;
6377         ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
6378
6379         ctx_pg = &ctx->tim_mem;
6380         ctx_pg->entries = ctx->qp_mem.entries;
6381         mem_size = ctx->tim_entry_size * ctx_pg->entries;
6382         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6383         if (rc)
6384                 return rc;
6385         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
6386
6387 skip_rdma:
6388         entries = ctx->qp_max_l2_entries + extra_qps;
6389         entries = roundup(entries, ctx->tqm_entries_multiple);
6390         entries = clamp_t(u32, entries, ctx->tqm_min_entries_per_ring,
6391                           ctx->tqm_max_entries_per_ring);
6392         for (i = 0; i < bp->max_q + 1; i++) {
6393                 ctx_pg = ctx->tqm_mem[i];
6394                 ctx_pg->entries = entries;
6395                 mem_size = ctx->tqm_entry_size * entries;
6396                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6397                 if (rc)
6398                         return rc;
6399                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
6400         }
6401         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
6402         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
6403         if (rc)
6404                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
6405                            rc);
6406         else
6407                 ctx->flags |= BNXT_CTX_FLAG_INITED;
6408
6409         return 0;
6410 }
6411
6412 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
6413 {
6414         struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6415         struct hwrm_func_resource_qcaps_input req = {0};
6416         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6417         int rc;
6418
6419         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
6420         req.fid = cpu_to_le16(0xffff);
6421
6422         mutex_lock(&bp->hwrm_cmd_lock);
6423         rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
6424                                        HWRM_CMD_TIMEOUT);
6425         if (rc) {
6426                 rc = -EIO;
6427                 goto hwrm_func_resc_qcaps_exit;
6428         }
6429
6430         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
6431         if (!all)
6432                 goto hwrm_func_resc_qcaps_exit;
6433
6434         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
6435         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6436         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
6437         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6438         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
6439         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6440         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
6441         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6442         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
6443         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
6444         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
6445         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6446         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
6447         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6448         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
6449         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6450
6451         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6452                 u16 max_msix = le16_to_cpu(resp->max_msix);
6453
6454                 hw_resc->max_nqs = max_msix;
6455                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
6456         }
6457
6458         if (BNXT_PF(bp)) {
6459                 struct bnxt_pf_info *pf = &bp->pf;
6460
6461                 pf->vf_resv_strategy =
6462                         le16_to_cpu(resp->vf_reservation_strategy);
6463                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
6464                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
6465         }
6466 hwrm_func_resc_qcaps_exit:
6467         mutex_unlock(&bp->hwrm_cmd_lock);
6468         return rc;
6469 }
6470
6471 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
6472 {
6473         int rc = 0;
6474         struct hwrm_func_qcaps_input req = {0};
6475         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6476         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6477         u32 flags;
6478
6479         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
6480         req.fid = cpu_to_le16(0xffff);
6481
6482         mutex_lock(&bp->hwrm_cmd_lock);
6483         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6484         if (rc)
6485                 goto hwrm_func_qcaps_exit;
6486
6487         flags = le32_to_cpu(resp->flags);
6488         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
6489                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
6490         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
6491                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
6492
6493         bp->tx_push_thresh = 0;
6494         if (flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED)
6495                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
6496
6497         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6498         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6499         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6500         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6501         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
6502         if (!hw_resc->max_hw_ring_grps)
6503                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
6504         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6505         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6506         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6507
6508         if (BNXT_PF(bp)) {
6509                 struct bnxt_pf_info *pf = &bp->pf;
6510
6511                 pf->fw_fid = le16_to_cpu(resp->fid);
6512                 pf->port_id = le16_to_cpu(resp->port_id);
6513                 bp->dev->dev_port = pf->port_id;
6514                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
6515                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
6516                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
6517                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
6518                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
6519                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
6520                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
6521                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
6522                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
6523                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
6524                         bp->flags |= BNXT_FLAG_WOL_CAP;
6525         } else {
6526 #ifdef CONFIG_BNXT_SRIOV
6527                 struct bnxt_vf_info *vf = &bp->vf;
6528
6529                 vf->fw_fid = le16_to_cpu(resp->fid);
6530                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
6531 #endif
6532         }
6533
6534 hwrm_func_qcaps_exit:
6535         mutex_unlock(&bp->hwrm_cmd_lock);
6536         return rc;
6537 }
6538
6539 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
6540
6541 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
6542 {
6543         int rc;
6544
6545         rc = __bnxt_hwrm_func_qcaps(bp);
6546         if (rc)
6547                 return rc;
6548         rc = bnxt_hwrm_queue_qportcfg(bp);
6549         if (rc) {
6550                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
6551                 return rc;
6552         }
6553         if (bp->hwrm_spec_code >= 0x10803) {
6554                 rc = bnxt_alloc_ctx_mem(bp);
6555                 if (rc)
6556                         return rc;
6557                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
6558                 if (!rc)
6559                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
6560         }
6561         return 0;
6562 }
6563
6564 static int bnxt_hwrm_func_reset(struct bnxt *bp)
6565 {
6566         struct hwrm_func_reset_input req = {0};
6567
6568         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
6569         req.enables = 0;
6570
6571         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
6572 }
6573
6574 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
6575 {
6576         int rc = 0;
6577         struct hwrm_queue_qportcfg_input req = {0};
6578         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
6579         u8 i, j, *qptr;
6580         bool no_rdma;
6581
6582         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
6583
6584         mutex_lock(&bp->hwrm_cmd_lock);
6585         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6586         if (rc)
6587                 goto qportcfg_exit;
6588
6589         if (!resp->max_configurable_queues) {
6590                 rc = -EINVAL;
6591                 goto qportcfg_exit;
6592         }
6593         bp->max_tc = resp->max_configurable_queues;
6594         bp->max_lltc = resp->max_configurable_lossless_queues;
6595         if (bp->max_tc > BNXT_MAX_QUEUE)
6596                 bp->max_tc = BNXT_MAX_QUEUE;
6597
6598         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
6599         qptr = &resp->queue_id0;
6600         for (i = 0, j = 0; i < bp->max_tc; i++) {
6601                 bp->q_info[j].queue_id = *qptr;
6602                 bp->q_ids[i] = *qptr++;
6603                 bp->q_info[j].queue_profile = *qptr++;
6604                 bp->tc_to_qidx[j] = j;
6605                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
6606                     (no_rdma && BNXT_PF(bp)))
6607                         j++;
6608         }
6609         bp->max_q = bp->max_tc;
6610         bp->max_tc = max_t(u8, j, 1);
6611
6612         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
6613                 bp->max_tc = 1;
6614
6615         if (bp->max_lltc > bp->max_tc)
6616                 bp->max_lltc = bp->max_tc;
6617
6618 qportcfg_exit:
6619         mutex_unlock(&bp->hwrm_cmd_lock);
6620         return rc;
6621 }
6622
6623 static int bnxt_hwrm_ver_get(struct bnxt *bp)
6624 {
6625         int rc;
6626         struct hwrm_ver_get_input req = {0};
6627         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
6628         u32 dev_caps_cfg;
6629
6630         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
6631         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
6632         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
6633         req.hwrm_intf_min = HWRM_VERSION_MINOR;
6634         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
6635         mutex_lock(&bp->hwrm_cmd_lock);
6636         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6637         if (rc)
6638                 goto hwrm_ver_get_exit;
6639
6640         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
6641
6642         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
6643                              resp->hwrm_intf_min_8b << 8 |
6644                              resp->hwrm_intf_upd_8b;
6645         if (resp->hwrm_intf_maj_8b < 1) {
6646                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
6647                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
6648                             resp->hwrm_intf_upd_8b);
6649                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
6650         }
6651         snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d",
6652                  resp->hwrm_fw_maj_8b, resp->hwrm_fw_min_8b,
6653                  resp->hwrm_fw_bld_8b, resp->hwrm_fw_rsvd_8b);
6654
6655         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
6656         if (!bp->hwrm_cmd_timeout)
6657                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
6658
6659         if (resp->hwrm_intf_maj_8b >= 1) {
6660                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
6661                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
6662         }
6663         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
6664                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
6665
6666         bp->chip_num = le16_to_cpu(resp->chip_num);
6667         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
6668             !resp->chip_metal)
6669                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
6670
6671         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
6672         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
6673             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
6674                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
6675
6676         if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
6677                 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
6678
6679         if (dev_caps_cfg &
6680             VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
6681                 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
6682
6683 hwrm_ver_get_exit:
6684         mutex_unlock(&bp->hwrm_cmd_lock);
6685         return rc;
6686 }
6687
6688 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
6689 {
6690         struct hwrm_fw_set_time_input req = {0};
6691         struct tm tm;
6692         time64_t now = ktime_get_real_seconds();
6693
6694         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
6695             bp->hwrm_spec_code < 0x10400)
6696                 return -EOPNOTSUPP;
6697
6698         time64_to_tm(now, 0, &tm);
6699         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
6700         req.year = cpu_to_le16(1900 + tm.tm_year);
6701         req.month = 1 + tm.tm_mon;
6702         req.day = tm.tm_mday;
6703         req.hour = tm.tm_hour;
6704         req.minute = tm.tm_min;
6705         req.second = tm.tm_sec;
6706         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6707 }
6708
6709 static int bnxt_hwrm_port_qstats(struct bnxt *bp)
6710 {
6711         int rc;
6712         struct bnxt_pf_info *pf = &bp->pf;
6713         struct hwrm_port_qstats_input req = {0};
6714
6715         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
6716                 return 0;
6717
6718         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
6719         req.port_id = cpu_to_le16(pf->port_id);
6720         req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
6721         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
6722         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6723         return rc;
6724 }
6725
6726 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp)
6727 {
6728         struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
6729         struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
6730         struct hwrm_port_qstats_ext_input req = {0};
6731         struct bnxt_pf_info *pf = &bp->pf;
6732         int rc;
6733
6734         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
6735                 return 0;
6736
6737         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
6738         req.port_id = cpu_to_le16(pf->port_id);
6739         req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
6740         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_ext_map);
6741         req.tx_stat_size = cpu_to_le16(sizeof(struct tx_port_stats_ext));
6742         req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_ext_map);
6743         mutex_lock(&bp->hwrm_cmd_lock);
6744         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6745         if (!rc) {
6746                 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
6747                 bp->fw_tx_stats_ext_size = le16_to_cpu(resp->tx_stat_size) / 8;
6748         } else {
6749                 bp->fw_rx_stats_ext_size = 0;
6750                 bp->fw_tx_stats_ext_size = 0;
6751         }
6752         if (bp->fw_tx_stats_ext_size <=
6753             offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
6754                 mutex_unlock(&bp->hwrm_cmd_lock);
6755                 bp->pri2cos_valid = 0;
6756                 return rc;
6757         }
6758
6759         bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
6760         req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
6761
6762         rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
6763         if (!rc) {
6764                 struct hwrm_queue_pri2cos_qcfg_output *resp2;
6765                 u8 *pri2cos;
6766                 int i, j;
6767
6768                 resp2 = bp->hwrm_cmd_resp_addr;
6769                 pri2cos = &resp2->pri0_cos_queue_id;
6770                 for (i = 0; i < 8; i++) {
6771                         u8 queue_id = pri2cos[i];
6772
6773                         for (j = 0; j < bp->max_q; j++) {
6774                                 if (bp->q_ids[j] == queue_id)
6775                                         bp->pri2cos[i] = j;
6776                         }
6777                 }
6778                 bp->pri2cos_valid = 1;
6779         }
6780         mutex_unlock(&bp->hwrm_cmd_lock);
6781         return rc;
6782 }
6783
6784 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
6785 {
6786         if (bp->vxlan_port_cnt) {
6787                 bnxt_hwrm_tunnel_dst_port_free(
6788                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
6789         }
6790         bp->vxlan_port_cnt = 0;
6791         if (bp->nge_port_cnt) {
6792                 bnxt_hwrm_tunnel_dst_port_free(
6793                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
6794         }
6795         bp->nge_port_cnt = 0;
6796 }
6797
6798 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
6799 {
6800         int rc, i;
6801         u32 tpa_flags = 0;
6802
6803         if (set_tpa)
6804                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
6805         for (i = 0; i < bp->nr_vnics; i++) {
6806                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
6807                 if (rc) {
6808                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
6809                                    i, rc);
6810                         return rc;
6811                 }
6812         }
6813         return 0;
6814 }
6815
6816 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
6817 {
6818         int i;
6819
6820         for (i = 0; i < bp->nr_vnics; i++)
6821                 bnxt_hwrm_vnic_set_rss(bp, i, false);
6822 }
6823
6824 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
6825                                     bool irq_re_init)
6826 {
6827         if (bp->vnic_info) {
6828                 bnxt_hwrm_clear_vnic_filter(bp);
6829                 /* clear all RSS setting before free vnic ctx */
6830                 bnxt_hwrm_clear_vnic_rss(bp);
6831                 bnxt_hwrm_vnic_ctx_free(bp);
6832                 /* before free the vnic, undo the vnic tpa settings */
6833                 if (bp->flags & BNXT_FLAG_TPA)
6834                         bnxt_set_tpa(bp, false);
6835                 bnxt_hwrm_vnic_free(bp);
6836         }
6837         bnxt_hwrm_ring_free(bp, close_path);
6838         bnxt_hwrm_ring_grp_free(bp);
6839         if (irq_re_init) {
6840                 bnxt_hwrm_stat_ctx_free(bp);
6841                 bnxt_hwrm_free_tunnel_ports(bp);
6842         }
6843 }
6844
6845 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
6846 {
6847         struct hwrm_func_cfg_input req = {0};
6848         int rc;
6849
6850         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
6851         req.fid = cpu_to_le16(0xffff);
6852         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
6853         if (br_mode == BRIDGE_MODE_VEB)
6854                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
6855         else if (br_mode == BRIDGE_MODE_VEPA)
6856                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
6857         else
6858                 return -EINVAL;
6859         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6860         if (rc)
6861                 rc = -EIO;
6862         return rc;
6863 }
6864
6865 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
6866 {
6867         struct hwrm_func_cfg_input req = {0};
6868         int rc;
6869
6870         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
6871                 return 0;
6872
6873         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
6874         req.fid = cpu_to_le16(0xffff);
6875         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
6876         req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
6877         if (size == 128)
6878                 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
6879
6880         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6881         if (rc)
6882                 rc = -EIO;
6883         return rc;
6884 }
6885
6886 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
6887 {
6888         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
6889         int rc;
6890
6891         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
6892                 goto skip_rss_ctx;
6893
6894         /* allocate context for vnic */
6895         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
6896         if (rc) {
6897                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
6898                            vnic_id, rc);
6899                 goto vnic_setup_err;
6900         }
6901         bp->rsscos_nr_ctxs++;
6902
6903         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
6904                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
6905                 if (rc) {
6906                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
6907                                    vnic_id, rc);
6908                         goto vnic_setup_err;
6909                 }
6910                 bp->rsscos_nr_ctxs++;
6911         }
6912
6913 skip_rss_ctx:
6914         /* configure default vnic, ring grp */
6915         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
6916         if (rc) {
6917                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
6918                            vnic_id, rc);
6919                 goto vnic_setup_err;
6920         }
6921
6922         /* Enable RSS hashing on vnic */
6923         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
6924         if (rc) {
6925                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
6926                            vnic_id, rc);
6927                 goto vnic_setup_err;
6928         }
6929
6930         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6931                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
6932                 if (rc) {
6933                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
6934                                    vnic_id, rc);
6935                 }
6936         }
6937
6938 vnic_setup_err:
6939         return rc;
6940 }
6941
6942 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
6943 {
6944         int rc, i, nr_ctxs;
6945
6946         nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
6947         for (i = 0; i < nr_ctxs; i++) {
6948                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
6949                 if (rc) {
6950                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
6951                                    vnic_id, i, rc);
6952                         break;
6953                 }
6954                 bp->rsscos_nr_ctxs++;
6955         }
6956         if (i < nr_ctxs)
6957                 return -ENOMEM;
6958
6959         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
6960         if (rc) {
6961                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
6962                            vnic_id, rc);
6963                 return rc;
6964         }
6965         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
6966         if (rc) {
6967                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
6968                            vnic_id, rc);
6969                 return rc;
6970         }
6971         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6972                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
6973                 if (rc) {
6974                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
6975                                    vnic_id, rc);
6976                 }
6977         }
6978         return rc;
6979 }
6980
6981 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
6982 {
6983         if (bp->flags & BNXT_FLAG_CHIP_P5)
6984                 return __bnxt_setup_vnic_p5(bp, vnic_id);
6985         else
6986                 return __bnxt_setup_vnic(bp, vnic_id);
6987 }
6988
6989 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
6990 {
6991 #ifdef CONFIG_RFS_ACCEL
6992         int i, rc = 0;
6993
6994         for (i = 0; i < bp->rx_nr_rings; i++) {
6995                 struct bnxt_vnic_info *vnic;
6996                 u16 vnic_id = i + 1;
6997                 u16 ring_id = i;
6998
6999                 if (vnic_id >= bp->nr_vnics)
7000                         break;
7001
7002                 vnic = &bp->vnic_info[vnic_id];
7003                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
7004                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
7005                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
7006                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
7007                 if (rc) {
7008                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7009                                    vnic_id, rc);
7010                         break;
7011                 }
7012                 rc = bnxt_setup_vnic(bp, vnic_id);
7013                 if (rc)
7014                         break;
7015         }
7016         return rc;
7017 #else
7018         return 0;
7019 #endif
7020 }
7021
7022 /* Allow PF and VF with default VLAN to be in promiscuous mode */
7023 static bool bnxt_promisc_ok(struct bnxt *bp)
7024 {
7025 #ifdef CONFIG_BNXT_SRIOV
7026         if (BNXT_VF(bp) && !bp->vf.vlan)
7027                 return false;
7028 #endif
7029         return true;
7030 }
7031
7032 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
7033 {
7034         unsigned int rc = 0;
7035
7036         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
7037         if (rc) {
7038                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
7039                            rc);
7040                 return rc;
7041         }
7042
7043         rc = bnxt_hwrm_vnic_cfg(bp, 1);
7044         if (rc) {
7045                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
7046                            rc);
7047                 return rc;
7048         }
7049         return rc;
7050 }
7051
7052 static int bnxt_cfg_rx_mode(struct bnxt *);
7053 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
7054
7055 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
7056 {
7057         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
7058         int rc = 0;
7059         unsigned int rx_nr_rings = bp->rx_nr_rings;
7060
7061         if (irq_re_init) {
7062                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
7063                 if (rc) {
7064                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
7065                                    rc);
7066                         goto err_out;
7067                 }
7068         }
7069
7070         rc = bnxt_hwrm_ring_alloc(bp);
7071         if (rc) {
7072                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
7073                 goto err_out;
7074         }
7075
7076         rc = bnxt_hwrm_ring_grp_alloc(bp);
7077         if (rc) {
7078                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
7079                 goto err_out;
7080         }
7081
7082         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7083                 rx_nr_rings--;
7084
7085         /* default vnic 0 */
7086         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
7087         if (rc) {
7088                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
7089                 goto err_out;
7090         }
7091
7092         rc = bnxt_setup_vnic(bp, 0);
7093         if (rc)
7094                 goto err_out;
7095
7096         if (bp->flags & BNXT_FLAG_RFS) {
7097                 rc = bnxt_alloc_rfs_vnics(bp);
7098                 if (rc)
7099                         goto err_out;
7100         }
7101
7102         if (bp->flags & BNXT_FLAG_TPA) {
7103                 rc = bnxt_set_tpa(bp, true);
7104                 if (rc)
7105                         goto err_out;
7106         }
7107
7108         if (BNXT_VF(bp))
7109                 bnxt_update_vf_mac(bp);
7110
7111         /* Filter for default vnic 0 */
7112         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
7113         if (rc) {
7114                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
7115                 goto err_out;
7116         }
7117         vnic->uc_filter_count = 1;
7118
7119         vnic->rx_mask = 0;
7120         if (bp->dev->flags & IFF_BROADCAST)
7121                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
7122
7123         if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
7124                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
7125
7126         if (bp->dev->flags & IFF_ALLMULTI) {
7127                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
7128                 vnic->mc_list_count = 0;
7129         } else {
7130                 u32 mask = 0;
7131
7132                 bnxt_mc_list_updated(bp, &mask);
7133                 vnic->rx_mask |= mask;
7134         }
7135
7136         rc = bnxt_cfg_rx_mode(bp);
7137         if (rc)
7138                 goto err_out;
7139
7140         rc = bnxt_hwrm_set_coal(bp);
7141         if (rc)
7142                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
7143                                 rc);
7144
7145         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7146                 rc = bnxt_setup_nitroa0_vnic(bp);
7147                 if (rc)
7148                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
7149                                    rc);
7150         }
7151
7152         if (BNXT_VF(bp)) {
7153                 bnxt_hwrm_func_qcfg(bp);
7154                 netdev_update_features(bp->dev);
7155         }
7156
7157         return 0;
7158
7159 err_out:
7160         bnxt_hwrm_resource_free(bp, 0, true);
7161
7162         return rc;
7163 }
7164
7165 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
7166 {
7167         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
7168         return 0;
7169 }
7170
7171 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
7172 {
7173         bnxt_init_cp_rings(bp);
7174         bnxt_init_rx_rings(bp);
7175         bnxt_init_tx_rings(bp);
7176         bnxt_init_ring_grps(bp, irq_re_init);
7177         bnxt_init_vnics(bp);
7178
7179         return bnxt_init_chip(bp, irq_re_init);
7180 }
7181
7182 static int bnxt_set_real_num_queues(struct bnxt *bp)
7183 {
7184         int rc;
7185         struct net_device *dev = bp->dev;
7186
7187         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
7188                                           bp->tx_nr_rings_xdp);
7189         if (rc)
7190                 return rc;
7191
7192         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
7193         if (rc)
7194                 return rc;
7195
7196 #ifdef CONFIG_RFS_ACCEL
7197         if (bp->flags & BNXT_FLAG_RFS)
7198                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
7199 #endif
7200
7201         return rc;
7202 }
7203
7204 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
7205                            bool shared)
7206 {
7207         int _rx = *rx, _tx = *tx;
7208
7209         if (shared) {
7210                 *rx = min_t(int, _rx, max);
7211                 *tx = min_t(int, _tx, max);
7212         } else {
7213                 if (max < 2)
7214                         return -ENOMEM;
7215
7216                 while (_rx + _tx > max) {
7217                         if (_rx > _tx && _rx > 1)
7218                                 _rx--;
7219                         else if (_tx > 1)
7220                                 _tx--;
7221                 }
7222                 *rx = _rx;
7223                 *tx = _tx;
7224         }
7225         return 0;
7226 }
7227
7228 static void bnxt_setup_msix(struct bnxt *bp)
7229 {
7230         const int len = sizeof(bp->irq_tbl[0].name);
7231         struct net_device *dev = bp->dev;
7232         int tcs, i;
7233
7234         tcs = netdev_get_num_tc(dev);
7235         if (tcs > 1) {
7236                 int i, off, count;
7237
7238                 for (i = 0; i < tcs; i++) {
7239                         count = bp->tx_nr_rings_per_tc;
7240                         off = i * count;
7241                         netdev_set_tc_queue(dev, i, count, off);
7242                 }
7243         }
7244
7245         for (i = 0; i < bp->cp_nr_rings; i++) {
7246                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7247                 char *attr;
7248
7249                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7250                         attr = "TxRx";
7251                 else if (i < bp->rx_nr_rings)
7252                         attr = "rx";
7253                 else
7254                         attr = "tx";
7255
7256                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
7257                          attr, i);
7258                 bp->irq_tbl[map_idx].handler = bnxt_msix;
7259         }
7260 }
7261
7262 static void bnxt_setup_inta(struct bnxt *bp)
7263 {
7264         const int len = sizeof(bp->irq_tbl[0].name);
7265
7266         if (netdev_get_num_tc(bp->dev))
7267                 netdev_reset_tc(bp->dev);
7268
7269         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
7270                  0);
7271         bp->irq_tbl[0].handler = bnxt_inta;
7272 }
7273
7274 static int bnxt_setup_int_mode(struct bnxt *bp)
7275 {
7276         int rc;
7277
7278         if (bp->flags & BNXT_FLAG_USING_MSIX)
7279                 bnxt_setup_msix(bp);
7280         else
7281                 bnxt_setup_inta(bp);
7282
7283         rc = bnxt_set_real_num_queues(bp);
7284         return rc;
7285 }
7286
7287 #ifdef CONFIG_RFS_ACCEL
7288 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
7289 {
7290         return bp->hw_resc.max_rsscos_ctxs;
7291 }
7292
7293 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
7294 {
7295         return bp->hw_resc.max_vnics;
7296 }
7297 #endif
7298
7299 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
7300 {
7301         return bp->hw_resc.max_stat_ctxs;
7302 }
7303
7304 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
7305 {
7306         return bp->hw_resc.max_cp_rings;
7307 }
7308
7309 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
7310 {
7311         unsigned int cp = bp->hw_resc.max_cp_rings;
7312
7313         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
7314                 cp -= bnxt_get_ulp_msix_num(bp);
7315
7316         return cp;
7317 }
7318
7319 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
7320 {
7321         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7322
7323         if (bp->flags & BNXT_FLAG_CHIP_P5)
7324                 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
7325
7326         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
7327 }
7328
7329 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
7330 {
7331         bp->hw_resc.max_irqs = max_irqs;
7332 }
7333
7334 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
7335 {
7336         unsigned int cp;
7337
7338         cp = bnxt_get_max_func_cp_rings_for_en(bp);
7339         if (bp->flags & BNXT_FLAG_CHIP_P5)
7340                 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
7341         else
7342                 return cp - bp->cp_nr_rings;
7343 }
7344
7345 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
7346 {
7347         unsigned int stat;
7348
7349         stat = bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_ulp_stat_ctxs(bp);
7350         stat -= bp->cp_nr_rings;
7351         return stat;
7352 }
7353
7354 int bnxt_get_avail_msix(struct bnxt *bp, int num)
7355 {
7356         int max_cp = bnxt_get_max_func_cp_rings(bp);
7357         int max_irq = bnxt_get_max_func_irqs(bp);
7358         int total_req = bp->cp_nr_rings + num;
7359         int max_idx, avail_msix;
7360
7361         max_idx = bp->total_irqs;
7362         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
7363                 max_idx = min_t(int, bp->total_irqs, max_cp);
7364         avail_msix = max_idx - bp->cp_nr_rings;
7365         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
7366                 return avail_msix;
7367
7368         if (max_irq < total_req) {
7369                 num = max_irq - bp->cp_nr_rings;
7370                 if (num <= 0)
7371                         return 0;
7372         }
7373         return num;
7374 }
7375
7376 static int bnxt_get_num_msix(struct bnxt *bp)
7377 {
7378         if (!BNXT_NEW_RM(bp))
7379                 return bnxt_get_max_func_irqs(bp);
7380
7381         return bnxt_nq_rings_in_use(bp);
7382 }
7383
7384 static int bnxt_init_msix(struct bnxt *bp)
7385 {
7386         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
7387         struct msix_entry *msix_ent;
7388
7389         total_vecs = bnxt_get_num_msix(bp);
7390         max = bnxt_get_max_func_irqs(bp);
7391         if (total_vecs > max)
7392                 total_vecs = max;
7393
7394         if (!total_vecs)
7395                 return 0;
7396
7397         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
7398         if (!msix_ent)
7399                 return -ENOMEM;
7400
7401         for (i = 0; i < total_vecs; i++) {
7402                 msix_ent[i].entry = i;
7403                 msix_ent[i].vector = 0;
7404         }
7405
7406         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
7407                 min = 2;
7408
7409         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
7410         ulp_msix = bnxt_get_ulp_msix_num(bp);
7411         if (total_vecs < 0 || total_vecs < ulp_msix) {
7412                 rc = -ENODEV;
7413                 goto msix_setup_exit;
7414         }
7415
7416         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
7417         if (bp->irq_tbl) {
7418                 for (i = 0; i < total_vecs; i++)
7419                         bp->irq_tbl[i].vector = msix_ent[i].vector;
7420
7421                 bp->total_irqs = total_vecs;
7422                 /* Trim rings based upon num of vectors allocated */
7423                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
7424                                      total_vecs - ulp_msix, min == 1);
7425                 if (rc)
7426                         goto msix_setup_exit;
7427
7428                 bp->cp_nr_rings = (min == 1) ?
7429                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
7430                                   bp->tx_nr_rings + bp->rx_nr_rings;
7431
7432         } else {
7433                 rc = -ENOMEM;
7434                 goto msix_setup_exit;
7435         }
7436         bp->flags |= BNXT_FLAG_USING_MSIX;
7437         kfree(msix_ent);
7438         return 0;
7439
7440 msix_setup_exit:
7441         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
7442         kfree(bp->irq_tbl);
7443         bp->irq_tbl = NULL;
7444         pci_disable_msix(bp->pdev);
7445         kfree(msix_ent);
7446         return rc;
7447 }
7448
7449 static int bnxt_init_inta(struct bnxt *bp)
7450 {
7451         bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
7452         if (!bp->irq_tbl)
7453                 return -ENOMEM;
7454
7455         bp->total_irqs = 1;
7456         bp->rx_nr_rings = 1;
7457         bp->tx_nr_rings = 1;
7458         bp->cp_nr_rings = 1;
7459         bp->flags |= BNXT_FLAG_SHARED_RINGS;
7460         bp->irq_tbl[0].vector = bp->pdev->irq;
7461         return 0;
7462 }
7463
7464 static int bnxt_init_int_mode(struct bnxt *bp)
7465 {
7466         int rc = 0;
7467
7468         if (bp->flags & BNXT_FLAG_MSIX_CAP)
7469                 rc = bnxt_init_msix(bp);
7470
7471         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
7472                 /* fallback to INTA */
7473                 rc = bnxt_init_inta(bp);
7474         }
7475         return rc;
7476 }
7477
7478 static void bnxt_clear_int_mode(struct bnxt *bp)
7479 {
7480         if (bp->flags & BNXT_FLAG_USING_MSIX)
7481                 pci_disable_msix(bp->pdev);
7482
7483         kfree(bp->irq_tbl);
7484         bp->irq_tbl = NULL;
7485         bp->flags &= ~BNXT_FLAG_USING_MSIX;
7486 }
7487
7488 int bnxt_reserve_rings(struct bnxt *bp)
7489 {
7490         int tcs = netdev_get_num_tc(bp->dev);
7491         bool reinit_irq = false;
7492         int rc;
7493
7494         if (!bnxt_need_reserve_rings(bp))
7495                 return 0;
7496
7497         if (BNXT_NEW_RM(bp) && (bnxt_get_num_msix(bp) != bp->total_irqs)) {
7498                 bnxt_ulp_irq_stop(bp);
7499                 bnxt_clear_int_mode(bp);
7500                 reinit_irq = true;
7501         }
7502         rc = __bnxt_reserve_rings(bp);
7503         if (reinit_irq) {
7504                 if (!rc)
7505                         rc = bnxt_init_int_mode(bp);
7506                 bnxt_ulp_irq_restart(bp, rc);
7507         }
7508         if (rc) {
7509                 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
7510                 return rc;
7511         }
7512         if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
7513                 netdev_err(bp->dev, "tx ring reservation failure\n");
7514                 netdev_reset_tc(bp->dev);
7515                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
7516                 return -ENOMEM;
7517         }
7518         return 0;
7519 }
7520
7521 static void bnxt_free_irq(struct bnxt *bp)
7522 {
7523         struct bnxt_irq *irq;
7524         int i;
7525
7526 #ifdef CONFIG_RFS_ACCEL
7527         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
7528         bp->dev->rx_cpu_rmap = NULL;
7529 #endif
7530         if (!bp->irq_tbl || !bp->bnapi)
7531                 return;
7532
7533         for (i = 0; i < bp->cp_nr_rings; i++) {
7534                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7535
7536                 irq = &bp->irq_tbl[map_idx];
7537                 if (irq->requested) {
7538                         if (irq->have_cpumask) {
7539                                 irq_set_affinity_hint(irq->vector, NULL);
7540                                 free_cpumask_var(irq->cpu_mask);
7541                                 irq->have_cpumask = 0;
7542                         }
7543                         free_irq(irq->vector, bp->bnapi[i]);
7544                 }
7545
7546                 irq->requested = 0;
7547         }
7548 }
7549
7550 static int bnxt_request_irq(struct bnxt *bp)
7551 {
7552         int i, j, rc = 0;
7553         unsigned long flags = 0;
7554 #ifdef CONFIG_RFS_ACCEL
7555         struct cpu_rmap *rmap;
7556 #endif
7557
7558         rc = bnxt_setup_int_mode(bp);
7559         if (rc) {
7560                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
7561                            rc);
7562                 return rc;
7563         }
7564 #ifdef CONFIG_RFS_ACCEL
7565         rmap = bp->dev->rx_cpu_rmap;
7566 #endif
7567         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
7568                 flags = IRQF_SHARED;
7569
7570         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
7571                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7572                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
7573
7574 #ifdef CONFIG_RFS_ACCEL
7575                 if (rmap && bp->bnapi[i]->rx_ring) {
7576                         rc = irq_cpu_rmap_add(rmap, irq->vector);
7577                         if (rc)
7578                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
7579                                             j);
7580                         j++;
7581                 }
7582 #endif
7583                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
7584                                  bp->bnapi[i]);
7585                 if (rc)
7586                         break;
7587
7588                 irq->requested = 1;
7589
7590                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
7591                         int numa_node = dev_to_node(&bp->pdev->dev);
7592
7593                         irq->have_cpumask = 1;
7594                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
7595                                         irq->cpu_mask);
7596                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
7597                         if (rc) {
7598                                 netdev_warn(bp->dev,
7599                                             "Set affinity failed, IRQ = %d\n",
7600                                             irq->vector);
7601                                 break;
7602                         }
7603                 }
7604         }
7605         return rc;
7606 }
7607
7608 static void bnxt_del_napi(struct bnxt *bp)
7609 {
7610         int i;
7611
7612         if (!bp->bnapi)
7613                 return;
7614
7615         for (i = 0; i < bp->cp_nr_rings; i++) {
7616                 struct bnxt_napi *bnapi = bp->bnapi[i];
7617
7618                 napi_hash_del(&bnapi->napi);
7619                 netif_napi_del(&bnapi->napi);
7620         }
7621         /* We called napi_hash_del() before netif_napi_del(), we need
7622          * to respect an RCU grace period before freeing napi structures.
7623          */
7624         synchronize_net();
7625 }
7626
7627 static void bnxt_init_napi(struct bnxt *bp)
7628 {
7629         int i;
7630         unsigned int cp_nr_rings = bp->cp_nr_rings;
7631         struct bnxt_napi *bnapi;
7632
7633         if (bp->flags & BNXT_FLAG_USING_MSIX) {
7634                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
7635
7636                 if (bp->flags & BNXT_FLAG_CHIP_P5)
7637                         poll_fn = bnxt_poll_p5;
7638                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7639                         cp_nr_rings--;
7640                 for (i = 0; i < cp_nr_rings; i++) {
7641                         bnapi = bp->bnapi[i];
7642                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
7643                 }
7644                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7645                         bnapi = bp->bnapi[cp_nr_rings];
7646                         netif_napi_add(bp->dev, &bnapi->napi,
7647                                        bnxt_poll_nitroa0, 64);
7648                 }
7649         } else {
7650                 bnapi = bp->bnapi[0];
7651                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
7652         }
7653 }
7654
7655 static void bnxt_disable_napi(struct bnxt *bp)
7656 {
7657         int i;
7658
7659         if (!bp->bnapi)
7660                 return;
7661
7662         for (i = 0; i < bp->cp_nr_rings; i++) {
7663                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
7664
7665                 if (bp->bnapi[i]->rx_ring)
7666                         cancel_work_sync(&cpr->dim.work);
7667
7668                 napi_disable(&bp->bnapi[i]->napi);
7669         }
7670 }
7671
7672 static void bnxt_enable_napi(struct bnxt *bp)
7673 {
7674         int i;
7675
7676         for (i = 0; i < bp->cp_nr_rings; i++) {
7677                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
7678                 bp->bnapi[i]->in_reset = false;
7679
7680                 if (bp->bnapi[i]->rx_ring) {
7681                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
7682                         cpr->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
7683                 }
7684                 napi_enable(&bp->bnapi[i]->napi);
7685         }
7686 }
7687
7688 void bnxt_tx_disable(struct bnxt *bp)
7689 {
7690         int i;
7691         struct bnxt_tx_ring_info *txr;
7692
7693         if (bp->tx_ring) {
7694                 for (i = 0; i < bp->tx_nr_rings; i++) {
7695                         txr = &bp->tx_ring[i];
7696                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
7697                 }
7698         }
7699         /* Stop all TX queues */
7700         netif_tx_disable(bp->dev);
7701         netif_carrier_off(bp->dev);
7702 }
7703
7704 void bnxt_tx_enable(struct bnxt *bp)
7705 {
7706         int i;
7707         struct bnxt_tx_ring_info *txr;
7708
7709         for (i = 0; i < bp->tx_nr_rings; i++) {
7710                 txr = &bp->tx_ring[i];
7711                 txr->dev_state = 0;
7712         }
7713         netif_tx_wake_all_queues(bp->dev);
7714         if (bp->link_info.link_up)
7715                 netif_carrier_on(bp->dev);
7716 }
7717
7718 static void bnxt_report_link(struct bnxt *bp)
7719 {
7720         if (bp->link_info.link_up) {
7721                 const char *duplex;
7722                 const char *flow_ctrl;
7723                 u32 speed;
7724                 u16 fec;
7725
7726                 netif_carrier_on(bp->dev);
7727                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
7728                         duplex = "full";
7729                 else
7730                         duplex = "half";
7731                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
7732                         flow_ctrl = "ON - receive & transmit";
7733                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
7734                         flow_ctrl = "ON - transmit";
7735                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
7736                         flow_ctrl = "ON - receive";
7737                 else
7738                         flow_ctrl = "none";
7739                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
7740                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
7741                             speed, duplex, flow_ctrl);
7742                 if (bp->flags & BNXT_FLAG_EEE_CAP)
7743                         netdev_info(bp->dev, "EEE is %s\n",
7744                                     bp->eee.eee_active ? "active" :
7745                                                          "not active");
7746                 fec = bp->link_info.fec_cfg;
7747                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
7748                         netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
7749                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
7750                                     (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
7751                                      (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
7752         } else {
7753                 netif_carrier_off(bp->dev);
7754                 netdev_err(bp->dev, "NIC Link is Down\n");
7755         }
7756 }
7757
7758 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
7759 {
7760         int rc = 0;
7761         struct hwrm_port_phy_qcaps_input req = {0};
7762         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7763         struct bnxt_link_info *link_info = &bp->link_info;
7764
7765         if (bp->hwrm_spec_code < 0x10201)
7766                 return 0;
7767
7768         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
7769
7770         mutex_lock(&bp->hwrm_cmd_lock);
7771         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7772         if (rc)
7773                 goto hwrm_phy_qcaps_exit;
7774
7775         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
7776                 struct ethtool_eee *eee = &bp->eee;
7777                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
7778
7779                 bp->flags |= BNXT_FLAG_EEE_CAP;
7780                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
7781                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
7782                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
7783                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
7784                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
7785         }
7786         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
7787                 if (bp->test_info)
7788                         bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
7789         }
7790         if (resp->supported_speeds_auto_mode)
7791                 link_info->support_auto_speeds =
7792                         le16_to_cpu(resp->supported_speeds_auto_mode);
7793
7794         bp->port_count = resp->port_cnt;
7795
7796 hwrm_phy_qcaps_exit:
7797         mutex_unlock(&bp->hwrm_cmd_lock);
7798         return rc;
7799 }
7800
7801 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
7802 {
7803         int rc = 0;
7804         struct bnxt_link_info *link_info = &bp->link_info;
7805         struct hwrm_port_phy_qcfg_input req = {0};
7806         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7807         u8 link_up = link_info->link_up;
7808         u16 diff;
7809
7810         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
7811
7812         mutex_lock(&bp->hwrm_cmd_lock);
7813         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7814         if (rc) {
7815                 mutex_unlock(&bp->hwrm_cmd_lock);
7816                 return rc;
7817         }
7818
7819         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
7820         link_info->phy_link_status = resp->link;
7821         link_info->duplex = resp->duplex_cfg;
7822         if (bp->hwrm_spec_code >= 0x10800)
7823                 link_info->duplex = resp->duplex_state;
7824         link_info->pause = resp->pause;
7825         link_info->auto_mode = resp->auto_mode;
7826         link_info->auto_pause_setting = resp->auto_pause;
7827         link_info->lp_pause = resp->link_partner_adv_pause;
7828         link_info->force_pause_setting = resp->force_pause;
7829         link_info->duplex_setting = resp->duplex_cfg;
7830         if (link_info->phy_link_status == BNXT_LINK_LINK)
7831                 link_info->link_speed = le16_to_cpu(resp->link_speed);
7832         else
7833                 link_info->link_speed = 0;
7834         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
7835         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
7836         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
7837         link_info->lp_auto_link_speeds =
7838                 le16_to_cpu(resp->link_partner_adv_speeds);
7839         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
7840         link_info->phy_ver[0] = resp->phy_maj;
7841         link_info->phy_ver[1] = resp->phy_min;
7842         link_info->phy_ver[2] = resp->phy_bld;
7843         link_info->media_type = resp->media_type;
7844         link_info->phy_type = resp->phy_type;
7845         link_info->transceiver = resp->xcvr_pkg_type;
7846         link_info->phy_addr = resp->eee_config_phy_addr &
7847                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
7848         link_info->module_status = resp->module_status;
7849
7850         if (bp->flags & BNXT_FLAG_EEE_CAP) {
7851                 struct ethtool_eee *eee = &bp->eee;
7852                 u16 fw_speeds;
7853
7854                 eee->eee_active = 0;
7855                 if (resp->eee_config_phy_addr &
7856                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
7857                         eee->eee_active = 1;
7858                         fw_speeds = le16_to_cpu(
7859                                 resp->link_partner_adv_eee_link_speed_mask);
7860                         eee->lp_advertised =
7861                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
7862                 }
7863
7864                 /* Pull initial EEE config */
7865                 if (!chng_link_state) {
7866                         if (resp->eee_config_phy_addr &
7867                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
7868                                 eee->eee_enabled = 1;
7869
7870                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
7871                         eee->advertised =
7872                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
7873
7874                         if (resp->eee_config_phy_addr &
7875                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
7876                                 __le32 tmr;
7877
7878                                 eee->tx_lpi_enabled = 1;
7879                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
7880                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
7881                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
7882                         }
7883                 }
7884         }
7885
7886         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
7887         if (bp->hwrm_spec_code >= 0x10504)
7888                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
7889
7890         /* TODO: need to add more logic to report VF link */
7891         if (chng_link_state) {
7892                 if (link_info->phy_link_status == BNXT_LINK_LINK)
7893                         link_info->link_up = 1;
7894                 else
7895                         link_info->link_up = 0;
7896                 if (link_up != link_info->link_up)
7897                         bnxt_report_link(bp);
7898         } else {
7899                 /* alwasy link down if not require to update link state */
7900                 link_info->link_up = 0;
7901         }
7902         mutex_unlock(&bp->hwrm_cmd_lock);
7903
7904         if (!BNXT_SINGLE_PF(bp))
7905                 return 0;
7906
7907         diff = link_info->support_auto_speeds ^ link_info->advertising;
7908         if ((link_info->support_auto_speeds | diff) !=
7909             link_info->support_auto_speeds) {
7910                 /* An advertised speed is no longer supported, so we need to
7911                  * update the advertisement settings.  Caller holds RTNL
7912                  * so we can modify link settings.
7913                  */
7914                 link_info->advertising = link_info->support_auto_speeds;
7915                 if (link_info->autoneg & BNXT_AUTONEG_SPEED)
7916                         bnxt_hwrm_set_link_setting(bp, true, false);
7917         }
7918         return 0;
7919 }
7920
7921 static void bnxt_get_port_module_status(struct bnxt *bp)
7922 {
7923         struct bnxt_link_info *link_info = &bp->link_info;
7924         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
7925         u8 module_status;
7926
7927         if (bnxt_update_link(bp, true))
7928                 return;
7929
7930         module_status = link_info->module_status;
7931         switch (module_status) {
7932         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
7933         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
7934         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
7935                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
7936                             bp->pf.port_id);
7937                 if (bp->hwrm_spec_code >= 0x10201) {
7938                         netdev_warn(bp->dev, "Module part number %s\n",
7939                                     resp->phy_vendor_partnumber);
7940                 }
7941                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
7942                         netdev_warn(bp->dev, "TX is disabled\n");
7943                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
7944                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
7945         }
7946 }
7947
7948 static void
7949 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
7950 {
7951         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
7952                 if (bp->hwrm_spec_code >= 0x10201)
7953                         req->auto_pause =
7954                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
7955                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
7956                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
7957                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
7958                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
7959                 req->enables |=
7960                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
7961         } else {
7962                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
7963                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
7964                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
7965                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
7966                 req->enables |=
7967                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
7968                 if (bp->hwrm_spec_code >= 0x10201) {
7969                         req->auto_pause = req->force_pause;
7970                         req->enables |= cpu_to_le32(
7971                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
7972                 }
7973         }
7974 }
7975
7976 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
7977                                       struct hwrm_port_phy_cfg_input *req)
7978 {
7979         u8 autoneg = bp->link_info.autoneg;
7980         u16 fw_link_speed = bp->link_info.req_link_speed;
7981         u16 advertising = bp->link_info.advertising;
7982
7983         if (autoneg & BNXT_AUTONEG_SPEED) {
7984                 req->auto_mode |=
7985                         PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
7986
7987                 req->enables |= cpu_to_le32(
7988                         PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
7989                 req->auto_link_speed_mask = cpu_to_le16(advertising);
7990
7991                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
7992                 req->flags |=
7993                         cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
7994         } else {
7995                 req->force_link_speed = cpu_to_le16(fw_link_speed);
7996                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
7997         }
7998
7999         /* tell chimp that the setting takes effect immediately */
8000         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
8001 }
8002
8003 int bnxt_hwrm_set_pause(struct bnxt *bp)
8004 {
8005         struct hwrm_port_phy_cfg_input req = {0};
8006         int rc;
8007
8008         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8009         bnxt_hwrm_set_pause_common(bp, &req);
8010
8011         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
8012             bp->link_info.force_link_chng)
8013                 bnxt_hwrm_set_link_common(bp, &req);
8014
8015         mutex_lock(&bp->hwrm_cmd_lock);
8016         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8017         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
8018                 /* since changing of pause setting doesn't trigger any link
8019                  * change event, the driver needs to update the current pause
8020                  * result upon successfully return of the phy_cfg command
8021                  */
8022                 bp->link_info.pause =
8023                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
8024                 bp->link_info.auto_pause_setting = 0;
8025                 if (!bp->link_info.force_link_chng)
8026                         bnxt_report_link(bp);
8027         }
8028         bp->link_info.force_link_chng = false;
8029         mutex_unlock(&bp->hwrm_cmd_lock);
8030         return rc;
8031 }
8032
8033 static void bnxt_hwrm_set_eee(struct bnxt *bp,
8034                               struct hwrm_port_phy_cfg_input *req)
8035 {
8036         struct ethtool_eee *eee = &bp->eee;
8037
8038         if (eee->eee_enabled) {
8039                 u16 eee_speeds;
8040                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
8041
8042                 if (eee->tx_lpi_enabled)
8043                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
8044                 else
8045                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
8046
8047                 req->flags |= cpu_to_le32(flags);
8048                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
8049                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
8050                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
8051         } else {
8052                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
8053         }
8054 }
8055
8056 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
8057 {
8058         struct hwrm_port_phy_cfg_input req = {0};
8059
8060         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8061         if (set_pause)
8062                 bnxt_hwrm_set_pause_common(bp, &req);
8063
8064         bnxt_hwrm_set_link_common(bp, &req);
8065
8066         if (set_eee)
8067                 bnxt_hwrm_set_eee(bp, &req);
8068         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8069 }
8070
8071 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
8072 {
8073         struct hwrm_port_phy_cfg_input req = {0};
8074
8075         if (!BNXT_SINGLE_PF(bp))
8076                 return 0;
8077
8078         if (pci_num_vf(bp->pdev))
8079                 return 0;
8080
8081         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8082         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
8083         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8084 }
8085
8086 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
8087 {
8088         struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
8089         struct hwrm_func_drv_if_change_input req = {0};
8090         bool resc_reinit = false;
8091         int rc;
8092
8093         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
8094                 return 0;
8095
8096         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
8097         if (up)
8098                 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
8099         mutex_lock(&bp->hwrm_cmd_lock);
8100         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8101         if (!rc && (resp->flags &
8102                     cpu_to_le32(FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)))
8103                 resc_reinit = true;
8104         mutex_unlock(&bp->hwrm_cmd_lock);
8105
8106         if (up && resc_reinit && BNXT_NEW_RM(bp)) {
8107                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8108
8109                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
8110                 hw_resc->resv_cp_rings = 0;
8111                 hw_resc->resv_stat_ctxs = 0;
8112                 hw_resc->resv_irqs = 0;
8113                 hw_resc->resv_tx_rings = 0;
8114                 hw_resc->resv_rx_rings = 0;
8115                 hw_resc->resv_hw_ring_grps = 0;
8116                 hw_resc->resv_vnics = 0;
8117                 bp->tx_nr_rings = 0;
8118                 bp->rx_nr_rings = 0;
8119         }
8120         return rc;
8121 }
8122
8123 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
8124 {
8125         struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8126         struct hwrm_port_led_qcaps_input req = {0};
8127         struct bnxt_pf_info *pf = &bp->pf;
8128         int rc;
8129
8130         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
8131                 return 0;
8132
8133         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
8134         req.port_id = cpu_to_le16(pf->port_id);
8135         mutex_lock(&bp->hwrm_cmd_lock);
8136         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8137         if (rc) {
8138                 mutex_unlock(&bp->hwrm_cmd_lock);
8139                 return rc;
8140         }
8141         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
8142                 int i;
8143
8144                 bp->num_leds = resp->num_leds;
8145                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
8146                                                  bp->num_leds);
8147                 for (i = 0; i < bp->num_leds; i++) {
8148                         struct bnxt_led_info *led = &bp->leds[i];
8149                         __le16 caps = led->led_state_caps;
8150
8151                         if (!led->led_group_id ||
8152                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
8153                                 bp->num_leds = 0;
8154                                 break;
8155                         }
8156                 }
8157         }
8158         mutex_unlock(&bp->hwrm_cmd_lock);
8159         return 0;
8160 }
8161
8162 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
8163 {
8164         struct hwrm_wol_filter_alloc_input req = {0};
8165         struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
8166         int rc;
8167
8168         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
8169         req.port_id = cpu_to_le16(bp->pf.port_id);
8170         req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
8171         req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
8172         memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
8173         mutex_lock(&bp->hwrm_cmd_lock);
8174         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8175         if (!rc)
8176                 bp->wol_filter_id = resp->wol_filter_id;
8177         mutex_unlock(&bp->hwrm_cmd_lock);
8178         return rc;
8179 }
8180
8181 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
8182 {
8183         struct hwrm_wol_filter_free_input req = {0};
8184         int rc;
8185
8186         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
8187         req.port_id = cpu_to_le16(bp->pf.port_id);
8188         req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
8189         req.wol_filter_id = bp->wol_filter_id;
8190         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8191         return rc;
8192 }
8193
8194 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
8195 {
8196         struct hwrm_wol_filter_qcfg_input req = {0};
8197         struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8198         u16 next_handle = 0;
8199         int rc;
8200
8201         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
8202         req.port_id = cpu_to_le16(bp->pf.port_id);
8203         req.handle = cpu_to_le16(handle);
8204         mutex_lock(&bp->hwrm_cmd_lock);
8205         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8206         if (!rc) {
8207                 next_handle = le16_to_cpu(resp->next_handle);
8208                 if (next_handle != 0) {
8209                         if (resp->wol_type ==
8210                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
8211                                 bp->wol = 1;
8212                                 bp->wol_filter_id = resp->wol_filter_id;
8213                         }
8214                 }
8215         }
8216         mutex_unlock(&bp->hwrm_cmd_lock);
8217         return next_handle;
8218 }
8219
8220 static void bnxt_get_wol_settings(struct bnxt *bp)
8221 {
8222         u16 handle = 0;
8223
8224         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
8225                 return;
8226
8227         do {
8228                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
8229         } while (handle && handle != 0xffff);
8230 }
8231
8232 #ifdef CONFIG_BNXT_HWMON
8233 static ssize_t bnxt_show_temp(struct device *dev,
8234                               struct device_attribute *devattr, char *buf)
8235 {
8236         struct hwrm_temp_monitor_query_input req = {0};
8237         struct hwrm_temp_monitor_query_output *resp;
8238         struct bnxt *bp = dev_get_drvdata(dev);
8239         u32 temp = 0;
8240
8241         resp = bp->hwrm_cmd_resp_addr;
8242         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
8243         mutex_lock(&bp->hwrm_cmd_lock);
8244         if (!_hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT))
8245                 temp = resp->temp * 1000; /* display millidegree */
8246         mutex_unlock(&bp->hwrm_cmd_lock);
8247
8248         return sprintf(buf, "%u\n", temp);
8249 }
8250 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
8251
8252 static struct attribute *bnxt_attrs[] = {
8253         &sensor_dev_attr_temp1_input.dev_attr.attr,
8254         NULL
8255 };
8256 ATTRIBUTE_GROUPS(bnxt);
8257
8258 static void bnxt_hwmon_close(struct bnxt *bp)
8259 {
8260         if (bp->hwmon_dev) {
8261                 hwmon_device_unregister(bp->hwmon_dev);
8262                 bp->hwmon_dev = NULL;
8263         }
8264 }
8265
8266 static void bnxt_hwmon_open(struct bnxt *bp)
8267 {
8268         struct pci_dev *pdev = bp->pdev;
8269
8270         bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
8271                                                           DRV_MODULE_NAME, bp,
8272                                                           bnxt_groups);
8273         if (IS_ERR(bp->hwmon_dev)) {
8274                 bp->hwmon_dev = NULL;
8275                 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
8276         }
8277 }
8278 #else
8279 static void bnxt_hwmon_close(struct bnxt *bp)
8280 {
8281 }
8282
8283 static void bnxt_hwmon_open(struct bnxt *bp)
8284 {
8285 }
8286 #endif
8287
8288 static bool bnxt_eee_config_ok(struct bnxt *bp)
8289 {
8290         struct ethtool_eee *eee = &bp->eee;
8291         struct bnxt_link_info *link_info = &bp->link_info;
8292
8293         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
8294                 return true;
8295
8296         if (eee->eee_enabled) {
8297                 u32 advertising =
8298                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
8299
8300                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
8301                         eee->eee_enabled = 0;
8302                         return false;
8303                 }
8304                 if (eee->advertised & ~advertising) {
8305                         eee->advertised = advertising & eee->supported;
8306                         return false;
8307                 }
8308         }
8309         return true;
8310 }
8311
8312 static int bnxt_update_phy_setting(struct bnxt *bp)
8313 {
8314         int rc;
8315         bool update_link = false;
8316         bool update_pause = false;
8317         bool update_eee = false;
8318         struct bnxt_link_info *link_info = &bp->link_info;
8319
8320         rc = bnxt_update_link(bp, true);
8321         if (rc) {
8322                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
8323                            rc);
8324                 return rc;
8325         }
8326         if (!BNXT_SINGLE_PF(bp))
8327                 return 0;
8328
8329         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
8330             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
8331             link_info->req_flow_ctrl)
8332                 update_pause = true;
8333         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
8334             link_info->force_pause_setting != link_info->req_flow_ctrl)
8335                 update_pause = true;
8336         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
8337                 if (BNXT_AUTO_MODE(link_info->auto_mode))
8338                         update_link = true;
8339                 if (link_info->req_link_speed != link_info->force_link_speed)
8340                         update_link = true;
8341                 if (link_info->req_duplex != link_info->duplex_setting)
8342                         update_link = true;
8343         } else {
8344                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
8345                         update_link = true;
8346                 if (link_info->advertising != link_info->auto_link_speeds)
8347                         update_link = true;
8348         }
8349
8350         /* The last close may have shutdown the link, so need to call
8351          * PHY_CFG to bring it back up.
8352          */
8353         if (!netif_carrier_ok(bp->dev))
8354                 update_link = true;
8355
8356         if (!bnxt_eee_config_ok(bp))
8357                 update_eee = true;
8358
8359         if (update_link)
8360                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
8361         else if (update_pause)
8362                 rc = bnxt_hwrm_set_pause(bp);
8363         if (rc) {
8364                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
8365                            rc);
8366                 return rc;
8367         }
8368
8369         return rc;
8370 }
8371
8372 /* Common routine to pre-map certain register block to different GRC window.
8373  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
8374  * in PF and 3 windows in VF that can be customized to map in different
8375  * register blocks.
8376  */
8377 static void bnxt_preset_reg_win(struct bnxt *bp)
8378 {
8379         if (BNXT_PF(bp)) {
8380                 /* CAG registers map to GRC window #4 */
8381                 writel(BNXT_CAG_REG_BASE,
8382                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
8383         }
8384 }
8385
8386 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
8387
8388 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
8389 {
8390         int rc = 0;
8391
8392         bnxt_preset_reg_win(bp);
8393         netif_carrier_off(bp->dev);
8394         if (irq_re_init) {
8395                 /* Reserve rings now if none were reserved at driver probe. */
8396                 rc = bnxt_init_dflt_ring_mode(bp);
8397                 if (rc) {
8398                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
8399                         return rc;
8400                 }
8401         }
8402         rc = bnxt_reserve_rings(bp);
8403         if (rc)
8404                 return rc;
8405         if ((bp->flags & BNXT_FLAG_RFS) &&
8406             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
8407                 /* disable RFS if falling back to INTA */
8408                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
8409                 bp->flags &= ~BNXT_FLAG_RFS;
8410         }
8411
8412         rc = bnxt_alloc_mem(bp, irq_re_init);
8413         if (rc) {
8414                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
8415                 goto open_err_free_mem;
8416         }
8417
8418         if (irq_re_init) {
8419                 bnxt_init_napi(bp);
8420                 rc = bnxt_request_irq(bp);
8421                 if (rc) {
8422                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
8423                         goto open_err_irq;
8424                 }
8425         }
8426
8427         bnxt_enable_napi(bp);
8428         bnxt_debug_dev_init(bp);
8429
8430         rc = bnxt_init_nic(bp, irq_re_init);
8431         if (rc) {
8432                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
8433                 goto open_err;
8434         }
8435
8436         if (link_re_init) {
8437                 mutex_lock(&bp->link_lock);
8438                 rc = bnxt_update_phy_setting(bp);
8439                 mutex_unlock(&bp->link_lock);
8440                 if (rc) {
8441                         netdev_warn(bp->dev, "failed to update phy settings\n");
8442                         if (BNXT_SINGLE_PF(bp)) {
8443                                 bp->link_info.phy_retry = true;
8444                                 bp->link_info.phy_retry_expires =
8445                                         jiffies + 5 * HZ;
8446                         }
8447                 }
8448         }
8449
8450         if (irq_re_init)
8451                 udp_tunnel_get_rx_info(bp->dev);
8452
8453         set_bit(BNXT_STATE_OPEN, &bp->state);
8454         bnxt_enable_int(bp);
8455         /* Enable TX queues */
8456         bnxt_tx_enable(bp);
8457         mod_timer(&bp->timer, jiffies + bp->current_interval);
8458         /* Poll link status and check for SFP+ module status */
8459         bnxt_get_port_module_status(bp);
8460
8461         /* VF-reps may need to be re-opened after the PF is re-opened */
8462         if (BNXT_PF(bp))
8463                 bnxt_vf_reps_open(bp);
8464         return 0;
8465
8466 open_err:
8467         bnxt_debug_dev_exit(bp);
8468         bnxt_disable_napi(bp);
8469
8470 open_err_irq:
8471         bnxt_del_napi(bp);
8472
8473 open_err_free_mem:
8474         bnxt_free_skbs(bp);
8475         bnxt_free_irq(bp);
8476         bnxt_free_mem(bp, true);
8477         return rc;
8478 }
8479
8480 /* rtnl_lock held */
8481 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
8482 {
8483         int rc = 0;
8484
8485         rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
8486         if (rc) {
8487                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
8488                 dev_close(bp->dev);
8489         }
8490         return rc;
8491 }
8492
8493 /* rtnl_lock held, open the NIC half way by allocating all resources, but
8494  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
8495  * self tests.
8496  */
8497 int bnxt_half_open_nic(struct bnxt *bp)
8498 {
8499         int rc = 0;
8500
8501         rc = bnxt_alloc_mem(bp, false);
8502         if (rc) {
8503                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
8504                 goto half_open_err;
8505         }
8506         rc = bnxt_init_nic(bp, false);
8507         if (rc) {
8508                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
8509                 goto half_open_err;
8510         }
8511         return 0;
8512
8513 half_open_err:
8514         bnxt_free_skbs(bp);
8515         bnxt_free_mem(bp, false);
8516         dev_close(bp->dev);
8517         return rc;
8518 }
8519
8520 /* rtnl_lock held, this call can only be made after a previous successful
8521  * call to bnxt_half_open_nic().
8522  */
8523 void bnxt_half_close_nic(struct bnxt *bp)
8524 {
8525         bnxt_hwrm_resource_free(bp, false, false);
8526         bnxt_free_skbs(bp);
8527         bnxt_free_mem(bp, false);
8528 }
8529
8530 static int bnxt_open(struct net_device *dev)
8531 {
8532         struct bnxt *bp = netdev_priv(dev);
8533         int rc;
8534
8535         bnxt_hwrm_if_change(bp, true);
8536         rc = __bnxt_open_nic(bp, true, true);
8537         if (rc)
8538                 bnxt_hwrm_if_change(bp, false);
8539
8540         bnxt_hwmon_open(bp);
8541
8542         return rc;
8543 }
8544
8545 static bool bnxt_drv_busy(struct bnxt *bp)
8546 {
8547         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
8548                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
8549 }
8550
8551 static void bnxt_get_ring_stats(struct bnxt *bp,
8552                                 struct rtnl_link_stats64 *stats);
8553
8554 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
8555                              bool link_re_init)
8556 {
8557         /* Close the VF-reps before closing PF */
8558         if (BNXT_PF(bp))
8559                 bnxt_vf_reps_close(bp);
8560
8561         /* Change device state to avoid TX queue wake up's */
8562         bnxt_tx_disable(bp);
8563
8564         clear_bit(BNXT_STATE_OPEN, &bp->state);
8565         smp_mb__after_atomic();
8566         while (bnxt_drv_busy(bp))
8567                 msleep(20);
8568
8569         /* Flush rings and and disable interrupts */
8570         bnxt_shutdown_nic(bp, irq_re_init);
8571
8572         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
8573
8574         bnxt_debug_dev_exit(bp);
8575         bnxt_disable_napi(bp);
8576         del_timer_sync(&bp->timer);
8577         bnxt_free_skbs(bp);
8578
8579         /* Save ring stats before shutdown */
8580         if (bp->bnapi)
8581                 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
8582         if (irq_re_init) {
8583                 bnxt_free_irq(bp);
8584                 bnxt_del_napi(bp);
8585         }
8586         bnxt_free_mem(bp, irq_re_init);
8587 }
8588
8589 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
8590 {
8591         int rc = 0;
8592
8593 #ifdef CONFIG_BNXT_SRIOV
8594         if (bp->sriov_cfg) {
8595                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
8596                                                       !bp->sriov_cfg,
8597                                                       BNXT_SRIOV_CFG_WAIT_TMO);
8598                 if (rc)
8599                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
8600         }
8601 #endif
8602         __bnxt_close_nic(bp, irq_re_init, link_re_init);
8603         return rc;
8604 }
8605
8606 static int bnxt_close(struct net_device *dev)
8607 {
8608         struct bnxt *bp = netdev_priv(dev);
8609
8610         bnxt_hwmon_close(bp);
8611         bnxt_close_nic(bp, true, true);
8612         bnxt_hwrm_shutdown_link(bp);
8613         bnxt_hwrm_if_change(bp, false);
8614         return 0;
8615 }
8616
8617 /* rtnl_lock held */
8618 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
8619 {
8620         switch (cmd) {
8621         case SIOCGMIIPHY:
8622                 /* fallthru */
8623         case SIOCGMIIREG: {
8624                 if (!netif_running(dev))
8625                         return -EAGAIN;
8626
8627                 return 0;
8628         }
8629
8630         case SIOCSMIIREG:
8631                 if (!netif_running(dev))
8632                         return -EAGAIN;
8633
8634                 return 0;
8635
8636         default:
8637                 /* do nothing */
8638                 break;
8639         }
8640         return -EOPNOTSUPP;
8641 }
8642
8643 static void bnxt_get_ring_stats(struct bnxt *bp,
8644                                 struct rtnl_link_stats64 *stats)
8645 {
8646         int i;
8647
8648
8649         for (i = 0; i < bp->cp_nr_rings; i++) {
8650                 struct bnxt_napi *bnapi = bp->bnapi[i];
8651                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
8652                 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
8653
8654                 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
8655                 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
8656                 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
8657
8658                 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
8659                 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
8660                 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
8661
8662                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
8663                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
8664                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
8665
8666                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
8667                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
8668                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
8669
8670                 stats->rx_missed_errors +=
8671                         le64_to_cpu(hw_stats->rx_discard_pkts);
8672
8673                 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
8674
8675                 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
8676         }
8677 }
8678
8679 static void bnxt_add_prev_stats(struct bnxt *bp,
8680                                 struct rtnl_link_stats64 *stats)
8681 {
8682         struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
8683
8684         stats->rx_packets += prev_stats->rx_packets;
8685         stats->tx_packets += prev_stats->tx_packets;
8686         stats->rx_bytes += prev_stats->rx_bytes;
8687         stats->tx_bytes += prev_stats->tx_bytes;
8688         stats->rx_missed_errors += prev_stats->rx_missed_errors;
8689         stats->multicast += prev_stats->multicast;
8690         stats->tx_dropped += prev_stats->tx_dropped;
8691 }
8692
8693 static void
8694 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
8695 {
8696         struct bnxt *bp = netdev_priv(dev);
8697
8698         set_bit(BNXT_STATE_READ_STATS, &bp->state);
8699         /* Make sure bnxt_close_nic() sees that we are reading stats before
8700          * we check the BNXT_STATE_OPEN flag.
8701          */
8702         smp_mb__after_atomic();
8703         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
8704                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
8705                 *stats = bp->net_stats_prev;
8706                 return;
8707         }
8708
8709         bnxt_get_ring_stats(bp, stats);
8710         bnxt_add_prev_stats(bp, stats);
8711
8712         if (bp->flags & BNXT_FLAG_PORT_STATS) {
8713                 struct rx_port_stats *rx = bp->hw_rx_port_stats;
8714                 struct tx_port_stats *tx = bp->hw_tx_port_stats;
8715
8716                 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
8717                 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
8718                 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
8719                                           le64_to_cpu(rx->rx_ovrsz_frames) +
8720                                           le64_to_cpu(rx->rx_runt_frames);
8721                 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
8722                                    le64_to_cpu(rx->rx_jbr_frames);
8723                 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
8724                 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
8725                 stats->tx_errors = le64_to_cpu(tx->tx_err);
8726         }
8727         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
8728 }
8729
8730 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
8731 {
8732         struct net_device *dev = bp->dev;
8733         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8734         struct netdev_hw_addr *ha;
8735         u8 *haddr;
8736         int mc_count = 0;
8737         bool update = false;
8738         int off = 0;
8739
8740         netdev_for_each_mc_addr(ha, dev) {
8741                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
8742                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8743                         vnic->mc_list_count = 0;
8744                         return false;
8745                 }
8746                 haddr = ha->addr;
8747                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
8748                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
8749                         update = true;
8750                 }
8751                 off += ETH_ALEN;
8752                 mc_count++;
8753         }
8754         if (mc_count)
8755                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
8756
8757         if (mc_count != vnic->mc_list_count) {
8758                 vnic->mc_list_count = mc_count;
8759                 update = true;
8760         }
8761         return update;
8762 }
8763
8764 static bool bnxt_uc_list_updated(struct bnxt *bp)
8765 {
8766         struct net_device *dev = bp->dev;
8767         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8768         struct netdev_hw_addr *ha;
8769         int off = 0;
8770
8771         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
8772                 return true;
8773
8774         netdev_for_each_uc_addr(ha, dev) {
8775                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
8776                         return true;
8777
8778                 off += ETH_ALEN;
8779         }
8780         return false;
8781 }
8782
8783 static void bnxt_set_rx_mode(struct net_device *dev)
8784 {
8785         struct bnxt *bp = netdev_priv(dev);
8786         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8787         u32 mask = vnic->rx_mask;
8788         bool mc_update = false;
8789         bool uc_update;
8790
8791         if (!netif_running(dev))
8792                 return;
8793
8794         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
8795                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
8796                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
8797                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
8798
8799         if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
8800                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8801
8802         uc_update = bnxt_uc_list_updated(bp);
8803
8804         if (dev->flags & IFF_BROADCAST)
8805                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8806         if (dev->flags & IFF_ALLMULTI) {
8807                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8808                 vnic->mc_list_count = 0;
8809         } else {
8810                 mc_update = bnxt_mc_list_updated(bp, &mask);
8811         }
8812
8813         if (mask != vnic->rx_mask || uc_update || mc_update) {
8814                 vnic->rx_mask = mask;
8815
8816                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
8817                 bnxt_queue_sp_work(bp);
8818         }
8819 }
8820
8821 static int bnxt_cfg_rx_mode(struct bnxt *bp)
8822 {
8823         struct net_device *dev = bp->dev;
8824         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8825         struct netdev_hw_addr *ha;
8826         int i, off = 0, rc;
8827         bool uc_update;
8828
8829         netif_addr_lock_bh(dev);
8830         uc_update = bnxt_uc_list_updated(bp);
8831         netif_addr_unlock_bh(dev);
8832
8833         if (!uc_update)
8834                 goto skip_uc;
8835
8836         mutex_lock(&bp->hwrm_cmd_lock);
8837         for (i = 1; i < vnic->uc_filter_count; i++) {
8838                 struct hwrm_cfa_l2_filter_free_input req = {0};
8839
8840                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
8841                                        -1);
8842
8843                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
8844
8845                 rc = _hwrm_send_message(bp, &req, sizeof(req),
8846                                         HWRM_CMD_TIMEOUT);
8847         }
8848         mutex_unlock(&bp->hwrm_cmd_lock);
8849
8850         vnic->uc_filter_count = 1;
8851
8852         netif_addr_lock_bh(dev);
8853         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
8854                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8855         } else {
8856                 netdev_for_each_uc_addr(ha, dev) {
8857                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
8858                         off += ETH_ALEN;
8859                         vnic->uc_filter_count++;
8860                 }
8861         }
8862         netif_addr_unlock_bh(dev);
8863
8864         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
8865                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
8866                 if (rc) {
8867                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
8868                                    rc);
8869                         vnic->uc_filter_count = i;
8870                         return rc;
8871                 }
8872         }
8873
8874 skip_uc:
8875         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
8876         if (rc)
8877                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n",
8878                            rc);
8879
8880         return rc;
8881 }
8882
8883 static bool bnxt_can_reserve_rings(struct bnxt *bp)
8884 {
8885 #ifdef CONFIG_BNXT_SRIOV
8886         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
8887                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8888
8889                 /* No minimum rings were provisioned by the PF.  Don't
8890                  * reserve rings by default when device is down.
8891                  */
8892                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
8893                         return true;
8894
8895                 if (!netif_running(bp->dev))
8896                         return false;
8897         }
8898 #endif
8899         return true;
8900 }
8901
8902 /* If the chip and firmware supports RFS */
8903 static bool bnxt_rfs_supported(struct bnxt *bp)
8904 {
8905         if (bp->flags & BNXT_FLAG_CHIP_P5)
8906                 return false;
8907         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
8908                 return true;
8909         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8910                 return true;
8911         return false;
8912 }
8913
8914 /* If runtime conditions support RFS */
8915 static bool bnxt_rfs_capable(struct bnxt *bp)
8916 {
8917 #ifdef CONFIG_RFS_ACCEL
8918         int vnics, max_vnics, max_rss_ctxs;
8919
8920         if (bp->flags & BNXT_FLAG_CHIP_P5)
8921                 return false;
8922         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
8923                 return false;
8924
8925         vnics = 1 + bp->rx_nr_rings;
8926         max_vnics = bnxt_get_max_func_vnics(bp);
8927         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
8928
8929         /* RSS contexts not a limiting factor */
8930         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8931                 max_rss_ctxs = max_vnics;
8932         if (vnics > max_vnics || vnics > max_rss_ctxs) {
8933                 if (bp->rx_nr_rings > 1)
8934                         netdev_warn(bp->dev,
8935                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
8936                                     min(max_rss_ctxs - 1, max_vnics - 1));
8937                 return false;
8938         }
8939
8940         if (!BNXT_NEW_RM(bp))
8941                 return true;
8942
8943         if (vnics == bp->hw_resc.resv_vnics)
8944                 return true;
8945
8946         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
8947         if (vnics <= bp->hw_resc.resv_vnics)
8948                 return true;
8949
8950         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
8951         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
8952         return false;
8953 #else
8954         return false;
8955 #endif
8956 }
8957
8958 static netdev_features_t bnxt_fix_features(struct net_device *dev,
8959                                            netdev_features_t features)
8960 {
8961         struct bnxt *bp = netdev_priv(dev);
8962
8963         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
8964                 features &= ~NETIF_F_NTUPLE;
8965
8966         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
8967                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
8968
8969         if (!(features & NETIF_F_GRO))
8970                 features &= ~NETIF_F_GRO_HW;
8971
8972         if (features & NETIF_F_GRO_HW)
8973                 features &= ~NETIF_F_LRO;
8974
8975         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
8976          * turned on or off together.
8977          */
8978         if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) !=
8979             (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) {
8980                 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
8981                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
8982                                       NETIF_F_HW_VLAN_STAG_RX);
8983                 else
8984                         features |= NETIF_F_HW_VLAN_CTAG_RX |
8985                                     NETIF_F_HW_VLAN_STAG_RX;
8986         }
8987 #ifdef CONFIG_BNXT_SRIOV
8988         if (BNXT_VF(bp)) {
8989                 if (bp->vf.vlan) {
8990                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
8991                                       NETIF_F_HW_VLAN_STAG_RX);
8992                 }
8993         }
8994 #endif
8995         return features;
8996 }
8997
8998 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
8999 {
9000         struct bnxt *bp = netdev_priv(dev);
9001         u32 flags = bp->flags;
9002         u32 changes;
9003         int rc = 0;
9004         bool re_init = false;
9005         bool update_tpa = false;
9006
9007         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
9008         if (features & NETIF_F_GRO_HW)
9009                 flags |= BNXT_FLAG_GRO;
9010         else if (features & NETIF_F_LRO)
9011                 flags |= BNXT_FLAG_LRO;
9012
9013         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
9014                 flags &= ~BNXT_FLAG_TPA;
9015
9016         if (features & NETIF_F_HW_VLAN_CTAG_RX)
9017                 flags |= BNXT_FLAG_STRIP_VLAN;
9018
9019         if (features & NETIF_F_NTUPLE)
9020                 flags |= BNXT_FLAG_RFS;
9021
9022         changes = flags ^ bp->flags;
9023         if (changes & BNXT_FLAG_TPA) {
9024                 update_tpa = true;
9025                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
9026                     (flags & BNXT_FLAG_TPA) == 0)
9027                         re_init = true;
9028         }
9029
9030         if (changes & ~BNXT_FLAG_TPA)
9031                 re_init = true;
9032
9033         if (flags != bp->flags) {
9034                 u32 old_flags = bp->flags;
9035
9036                 bp->flags = flags;
9037
9038                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9039                         if (update_tpa)
9040                                 bnxt_set_ring_params(bp);
9041                         return rc;
9042                 }
9043
9044                 if (re_init) {
9045                         bnxt_close_nic(bp, false, false);
9046                         if (update_tpa)
9047                                 bnxt_set_ring_params(bp);
9048
9049                         return bnxt_open_nic(bp, false, false);
9050                 }
9051                 if (update_tpa) {
9052                         rc = bnxt_set_tpa(bp,
9053                                           (flags & BNXT_FLAG_TPA) ?
9054                                           true : false);
9055                         if (rc)
9056                                 bp->flags = old_flags;
9057                 }
9058         }
9059         return rc;
9060 }
9061
9062 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
9063                                        u32 ring_id, u32 *prod, u32 *cons)
9064 {
9065         struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
9066         struct hwrm_dbg_ring_info_get_input req = {0};
9067         int rc;
9068
9069         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
9070         req.ring_type = ring_type;
9071         req.fw_ring_id = cpu_to_le32(ring_id);
9072         mutex_lock(&bp->hwrm_cmd_lock);
9073         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9074         if (!rc) {
9075                 *prod = le32_to_cpu(resp->producer_index);
9076                 *cons = le32_to_cpu(resp->consumer_index);
9077         }
9078         mutex_unlock(&bp->hwrm_cmd_lock);
9079         return rc;
9080 }
9081
9082 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
9083 {
9084         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
9085         int i = bnapi->index;
9086
9087         if (!txr)
9088                 return;
9089
9090         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
9091                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
9092                     txr->tx_cons);
9093 }
9094
9095 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
9096 {
9097         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
9098         int i = bnapi->index;
9099
9100         if (!rxr)
9101                 return;
9102
9103         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
9104                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
9105                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
9106                     rxr->rx_sw_agg_prod);
9107 }
9108
9109 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
9110 {
9111         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9112         int i = bnapi->index;
9113
9114         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
9115                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
9116 }
9117
9118 static void bnxt_dbg_dump_states(struct bnxt *bp)
9119 {
9120         int i;
9121         struct bnxt_napi *bnapi;
9122
9123         for (i = 0; i < bp->cp_nr_rings; i++) {
9124                 bnapi = bp->bnapi[i];
9125                 if (netif_msg_drv(bp)) {
9126                         bnxt_dump_tx_sw_state(bnapi);
9127                         bnxt_dump_rx_sw_state(bnapi);
9128                         bnxt_dump_cp_sw_state(bnapi);
9129                 }
9130         }
9131 }
9132
9133 static void bnxt_reset_task(struct bnxt *bp, bool silent)
9134 {
9135         if (!silent)
9136                 bnxt_dbg_dump_states(bp);
9137         if (netif_running(bp->dev)) {
9138                 int rc;
9139
9140                 if (!silent)
9141                         bnxt_ulp_stop(bp);
9142                 bnxt_close_nic(bp, false, false);
9143                 rc = bnxt_open_nic(bp, false, false);
9144                 if (!silent && !rc)
9145                         bnxt_ulp_start(bp);
9146         }
9147 }
9148
9149 static void bnxt_tx_timeout(struct net_device *dev)
9150 {
9151         struct bnxt *bp = netdev_priv(dev);
9152
9153         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
9154         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
9155         bnxt_queue_sp_work(bp);
9156 }
9157
9158 static void bnxt_timer(struct timer_list *t)
9159 {
9160         struct bnxt *bp = from_timer(bp, t, timer);
9161         struct net_device *dev = bp->dev;
9162
9163         if (!netif_running(dev))
9164                 return;
9165
9166         if (atomic_read(&bp->intr_sem) != 0)
9167                 goto bnxt_restart_timer;
9168
9169         if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) &&
9170             bp->stats_coal_ticks) {
9171                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
9172                 bnxt_queue_sp_work(bp);
9173         }
9174
9175         if (bnxt_tc_flower_enabled(bp)) {
9176                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
9177                 bnxt_queue_sp_work(bp);
9178         }
9179
9180         if (bp->link_info.phy_retry) {
9181                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
9182                         bp->link_info.phy_retry = 0;
9183                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
9184                 } else {
9185                         set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
9186                         bnxt_queue_sp_work(bp);
9187                 }
9188         }
9189
9190         if ((bp->flags & BNXT_FLAG_CHIP_P5) && netif_carrier_ok(dev)) {
9191                 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
9192                 bnxt_queue_sp_work(bp);
9193         }
9194 bnxt_restart_timer:
9195         mod_timer(&bp->timer, jiffies + bp->current_interval);
9196 }
9197
9198 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
9199 {
9200         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
9201          * set.  If the device is being closed, bnxt_close() may be holding
9202          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
9203          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
9204          */
9205         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9206         rtnl_lock();
9207 }
9208
9209 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
9210 {
9211         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9212         rtnl_unlock();
9213 }
9214
9215 /* Only called from bnxt_sp_task() */
9216 static void bnxt_reset(struct bnxt *bp, bool silent)
9217 {
9218         bnxt_rtnl_lock_sp(bp);
9219         if (test_bit(BNXT_STATE_OPEN, &bp->state))
9220                 bnxt_reset_task(bp, silent);
9221         bnxt_rtnl_unlock_sp(bp);
9222 }
9223
9224 static void bnxt_chk_missed_irq(struct bnxt *bp)
9225 {
9226         int i;
9227
9228         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9229                 return;
9230
9231         for (i = 0; i < bp->cp_nr_rings; i++) {
9232                 struct bnxt_napi *bnapi = bp->bnapi[i];
9233                 struct bnxt_cp_ring_info *cpr;
9234                 u32 fw_ring_id;
9235                 int j;
9236
9237                 if (!bnapi)
9238                         continue;
9239
9240                 cpr = &bnapi->cp_ring;
9241                 for (j = 0; j < 2; j++) {
9242                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
9243                         u32 val[2];
9244
9245                         if (!cpr2 || cpr2->has_more_work ||
9246                             !bnxt_has_work(bp, cpr2))
9247                                 continue;
9248
9249                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
9250                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
9251                                 continue;
9252                         }
9253                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
9254                         bnxt_dbg_hwrm_ring_info_get(bp,
9255                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
9256                                 fw_ring_id, &val[0], &val[1]);
9257                         cpr->missed_irqs++;
9258                 }
9259         }
9260 }
9261
9262 static void bnxt_cfg_ntp_filters(struct bnxt *);
9263
9264 static void bnxt_sp_task(struct work_struct *work)
9265 {
9266         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
9267
9268         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9269         smp_mb__after_atomic();
9270         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9271                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9272                 return;
9273         }
9274
9275         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
9276                 bnxt_cfg_rx_mode(bp);
9277
9278         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
9279                 bnxt_cfg_ntp_filters(bp);
9280         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
9281                 bnxt_hwrm_exec_fwd_req(bp);
9282         if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
9283                 bnxt_hwrm_tunnel_dst_port_alloc(
9284                         bp, bp->vxlan_port,
9285                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
9286         }
9287         if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
9288                 bnxt_hwrm_tunnel_dst_port_free(
9289                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
9290         }
9291         if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) {
9292                 bnxt_hwrm_tunnel_dst_port_alloc(
9293                         bp, bp->nge_port,
9294                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
9295         }
9296         if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) {
9297                 bnxt_hwrm_tunnel_dst_port_free(
9298                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
9299         }
9300         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
9301                 bnxt_hwrm_port_qstats(bp);
9302                 bnxt_hwrm_port_qstats_ext(bp);
9303         }
9304
9305         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
9306                 int rc;
9307
9308                 mutex_lock(&bp->link_lock);
9309                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
9310                                        &bp->sp_event))
9311                         bnxt_hwrm_phy_qcaps(bp);
9312
9313                 rc = bnxt_update_link(bp, true);
9314                 mutex_unlock(&bp->link_lock);
9315                 if (rc)
9316                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
9317                                    rc);
9318         }
9319         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
9320                 int rc;
9321
9322                 mutex_lock(&bp->link_lock);
9323                 rc = bnxt_update_phy_setting(bp);
9324                 mutex_unlock(&bp->link_lock);
9325                 if (rc) {
9326                         netdev_warn(bp->dev, "update phy settings retry failed\n");
9327                 } else {
9328                         bp->link_info.phy_retry = false;
9329                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
9330                 }
9331         }
9332         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
9333                 mutex_lock(&bp->link_lock);
9334                 bnxt_get_port_module_status(bp);
9335                 mutex_unlock(&bp->link_lock);
9336         }
9337
9338         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
9339                 bnxt_tc_flow_stats_work(bp);
9340
9341         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
9342                 bnxt_chk_missed_irq(bp);
9343
9344         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
9345          * must be the last functions to be called before exiting.
9346          */
9347         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
9348                 bnxt_reset(bp, false);
9349
9350         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
9351                 bnxt_reset(bp, true);
9352
9353         smp_mb__before_atomic();
9354         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9355 }
9356
9357 /* Under rtnl_lock */
9358 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
9359                      int tx_xdp)
9360 {
9361         int max_rx, max_tx, tx_sets = 1;
9362         int tx_rings_needed, stats;
9363         int rx_rings = rx;
9364         int cp, vnics, rc;
9365
9366         if (tcs)
9367                 tx_sets = tcs;
9368
9369         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
9370         if (rc)
9371                 return rc;
9372
9373         if (max_rx < rx)
9374                 return -ENOMEM;
9375
9376         tx_rings_needed = tx * tx_sets + tx_xdp;
9377         if (max_tx < tx_rings_needed)
9378                 return -ENOMEM;
9379
9380         vnics = 1;
9381         if (bp->flags & BNXT_FLAG_RFS)
9382                 vnics += rx_rings;
9383
9384         if (bp->flags & BNXT_FLAG_AGG_RINGS)
9385                 rx_rings <<= 1;
9386         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
9387         stats = cp;
9388         if (BNXT_NEW_RM(bp)) {
9389                 cp += bnxt_get_ulp_msix_num(bp);
9390                 stats += bnxt_get_ulp_stat_ctxs(bp);
9391         }
9392         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
9393                                      stats, vnics);
9394 }
9395
9396 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
9397 {
9398         if (bp->bar2) {
9399                 pci_iounmap(pdev, bp->bar2);
9400                 bp->bar2 = NULL;
9401         }
9402
9403         if (bp->bar1) {
9404                 pci_iounmap(pdev, bp->bar1);
9405                 bp->bar1 = NULL;
9406         }
9407
9408         if (bp->bar0) {
9409                 pci_iounmap(pdev, bp->bar0);
9410                 bp->bar0 = NULL;
9411         }
9412 }
9413
9414 static void bnxt_cleanup_pci(struct bnxt *bp)
9415 {
9416         bnxt_unmap_bars(bp, bp->pdev);
9417         pci_release_regions(bp->pdev);
9418         pci_disable_device(bp->pdev);
9419 }
9420
9421 static void bnxt_init_dflt_coal(struct bnxt *bp)
9422 {
9423         struct bnxt_coal *coal;
9424
9425         /* Tick values in micro seconds.
9426          * 1 coal_buf x bufs_per_record = 1 completion record.
9427          */
9428         coal = &bp->rx_coal;
9429         coal->coal_ticks = 10;
9430         coal->coal_bufs = 30;
9431         coal->coal_ticks_irq = 1;
9432         coal->coal_bufs_irq = 2;
9433         coal->idle_thresh = 50;
9434         coal->bufs_per_record = 2;
9435         coal->budget = 64;              /* NAPI budget */
9436
9437         coal = &bp->tx_coal;
9438         coal->coal_ticks = 28;
9439         coal->coal_bufs = 30;
9440         coal->coal_ticks_irq = 2;
9441         coal->coal_bufs_irq = 2;
9442         coal->bufs_per_record = 1;
9443
9444         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
9445 }
9446
9447 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
9448 {
9449         int rc;
9450         struct bnxt *bp = netdev_priv(dev);
9451
9452         SET_NETDEV_DEV(dev, &pdev->dev);
9453
9454         /* enable device (incl. PCI PM wakeup), and bus-mastering */
9455         rc = pci_enable_device(pdev);
9456         if (rc) {
9457                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
9458                 goto init_err;
9459         }
9460
9461         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
9462                 dev_err(&pdev->dev,
9463                         "Cannot find PCI device base address, aborting\n");
9464                 rc = -ENODEV;
9465                 goto init_err_disable;
9466         }
9467
9468         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
9469         if (rc) {
9470                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
9471                 goto init_err_disable;
9472         }
9473
9474         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
9475             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
9476                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
9477                 goto init_err_disable;
9478         }
9479
9480         pci_set_master(pdev);
9481
9482         bp->dev = dev;
9483         bp->pdev = pdev;
9484
9485         bp->bar0 = pci_ioremap_bar(pdev, 0);
9486         if (!bp->bar0) {
9487                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
9488                 rc = -ENOMEM;
9489                 goto init_err_release;
9490         }
9491
9492         bp->bar1 = pci_ioremap_bar(pdev, 2);
9493         if (!bp->bar1) {
9494                 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
9495                 rc = -ENOMEM;
9496                 goto init_err_release;
9497         }
9498
9499         bp->bar2 = pci_ioremap_bar(pdev, 4);
9500         if (!bp->bar2) {
9501                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
9502                 rc = -ENOMEM;
9503                 goto init_err_release;
9504         }
9505
9506         pci_enable_pcie_error_reporting(pdev);
9507
9508         INIT_WORK(&bp->sp_task, bnxt_sp_task);
9509
9510         spin_lock_init(&bp->ntp_fltr_lock);
9511 #if BITS_PER_LONG == 32
9512         spin_lock_init(&bp->db_lock);
9513 #endif
9514
9515         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
9516         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
9517
9518         bnxt_init_dflt_coal(bp);
9519
9520         timer_setup(&bp->timer, bnxt_timer, 0);
9521         bp->current_interval = BNXT_TIMER_INTERVAL;
9522
9523         clear_bit(BNXT_STATE_OPEN, &bp->state);
9524         return 0;
9525
9526 init_err_release:
9527         bnxt_unmap_bars(bp, pdev);
9528         pci_release_regions(pdev);
9529
9530 init_err_disable:
9531         pci_disable_device(pdev);
9532
9533 init_err:
9534         return rc;
9535 }
9536
9537 /* rtnl_lock held */
9538 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
9539 {
9540         struct sockaddr *addr = p;
9541         struct bnxt *bp = netdev_priv(dev);
9542         int rc = 0;
9543
9544         if (!is_valid_ether_addr(addr->sa_data))
9545                 return -EADDRNOTAVAIL;
9546
9547         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
9548                 return 0;
9549
9550         rc = bnxt_approve_mac(bp, addr->sa_data, true);
9551         if (rc)
9552                 return rc;
9553
9554         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
9555         if (netif_running(dev)) {
9556                 bnxt_close_nic(bp, false, false);
9557                 rc = bnxt_open_nic(bp, false, false);
9558         }
9559
9560         return rc;
9561 }
9562
9563 /* rtnl_lock held */
9564 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
9565 {
9566         struct bnxt *bp = netdev_priv(dev);
9567
9568         if (netif_running(dev))
9569                 bnxt_close_nic(bp, false, false);
9570
9571         dev->mtu = new_mtu;
9572         bnxt_set_ring_params(bp);
9573
9574         if (netif_running(dev))
9575                 return bnxt_open_nic(bp, false, false);
9576
9577         return 0;
9578 }
9579
9580 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
9581 {
9582         struct bnxt *bp = netdev_priv(dev);
9583         bool sh = false;
9584         int rc;
9585
9586         if (tc > bp->max_tc) {
9587                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
9588                            tc, bp->max_tc);
9589                 return -EINVAL;
9590         }
9591
9592         if (netdev_get_num_tc(dev) == tc)
9593                 return 0;
9594
9595         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
9596                 sh = true;
9597
9598         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
9599                               sh, tc, bp->tx_nr_rings_xdp);
9600         if (rc)
9601                 return rc;
9602
9603         /* Needs to close the device and do hw resource re-allocations */
9604         if (netif_running(bp->dev))
9605                 bnxt_close_nic(bp, true, false);
9606
9607         if (tc) {
9608                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
9609                 netdev_set_num_tc(dev, tc);
9610         } else {
9611                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
9612                 netdev_reset_tc(dev);
9613         }
9614         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
9615         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
9616                                bp->tx_nr_rings + bp->rx_nr_rings;
9617
9618         if (netif_running(bp->dev))
9619                 return bnxt_open_nic(bp, true, false);
9620
9621         return 0;
9622 }
9623
9624 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
9625                                   void *cb_priv)
9626 {
9627         struct bnxt *bp = cb_priv;
9628
9629         if (!bnxt_tc_flower_enabled(bp) ||
9630             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
9631                 return -EOPNOTSUPP;
9632
9633         switch (type) {
9634         case TC_SETUP_CLSFLOWER:
9635                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
9636         default:
9637                 return -EOPNOTSUPP;
9638         }
9639 }
9640
9641 static int bnxt_setup_tc_block(struct net_device *dev,
9642                                struct tc_block_offload *f)
9643 {
9644         struct bnxt *bp = netdev_priv(dev);
9645
9646         if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
9647                 return -EOPNOTSUPP;
9648
9649         switch (f->command) {
9650         case TC_BLOCK_BIND:
9651                 return tcf_block_cb_register(f->block, bnxt_setup_tc_block_cb,
9652                                              bp, bp, f->extack);
9653         case TC_BLOCK_UNBIND:
9654                 tcf_block_cb_unregister(f->block, bnxt_setup_tc_block_cb, bp);
9655                 return 0;
9656         default:
9657                 return -EOPNOTSUPP;
9658         }
9659 }
9660
9661 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
9662                          void *type_data)
9663 {
9664         switch (type) {
9665         case TC_SETUP_BLOCK:
9666                 return bnxt_setup_tc_block(dev, type_data);
9667         case TC_SETUP_QDISC_MQPRIO: {
9668                 struct tc_mqprio_qopt *mqprio = type_data;
9669
9670                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
9671
9672                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
9673         }
9674         default:
9675                 return -EOPNOTSUPP;
9676         }
9677 }
9678
9679 #ifdef CONFIG_RFS_ACCEL
9680 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
9681                             struct bnxt_ntuple_filter *f2)
9682 {
9683         struct flow_keys *keys1 = &f1->fkeys;
9684         struct flow_keys *keys2 = &f2->fkeys;
9685
9686         if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
9687             keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
9688             keys1->ports.ports == keys2->ports.ports &&
9689             keys1->basic.ip_proto == keys2->basic.ip_proto &&
9690             keys1->basic.n_proto == keys2->basic.n_proto &&
9691             keys1->control.flags == keys2->control.flags &&
9692             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
9693             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
9694                 return true;
9695
9696         return false;
9697 }
9698
9699 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
9700                               u16 rxq_index, u32 flow_id)
9701 {
9702         struct bnxt *bp = netdev_priv(dev);
9703         struct bnxt_ntuple_filter *fltr, *new_fltr;
9704         struct flow_keys *fkeys;
9705         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
9706         int rc = 0, idx, bit_id, l2_idx = 0;
9707         struct hlist_head *head;
9708
9709         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
9710                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9711                 int off = 0, j;
9712
9713                 netif_addr_lock_bh(dev);
9714                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
9715                         if (ether_addr_equal(eth->h_dest,
9716                                              vnic->uc_list + off)) {
9717                                 l2_idx = j + 1;
9718                                 break;
9719                         }
9720                 }
9721                 netif_addr_unlock_bh(dev);
9722                 if (!l2_idx)
9723                         return -EINVAL;
9724         }
9725         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
9726         if (!new_fltr)
9727                 return -ENOMEM;
9728
9729         fkeys = &new_fltr->fkeys;
9730         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
9731                 rc = -EPROTONOSUPPORT;
9732                 goto err_free;
9733         }
9734
9735         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
9736              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
9737             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
9738              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
9739                 rc = -EPROTONOSUPPORT;
9740                 goto err_free;
9741         }
9742         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
9743             bp->hwrm_spec_code < 0x10601) {
9744                 rc = -EPROTONOSUPPORT;
9745                 goto err_free;
9746         }
9747         if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) &&
9748             bp->hwrm_spec_code < 0x10601) {
9749                 rc = -EPROTONOSUPPORT;
9750                 goto err_free;
9751         }
9752
9753         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
9754         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
9755
9756         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
9757         head = &bp->ntp_fltr_hash_tbl[idx];
9758         rcu_read_lock();
9759         hlist_for_each_entry_rcu(fltr, head, hash) {
9760                 if (bnxt_fltr_match(fltr, new_fltr)) {
9761                         rcu_read_unlock();
9762                         rc = 0;
9763                         goto err_free;
9764                 }
9765         }
9766         rcu_read_unlock();
9767
9768         spin_lock_bh(&bp->ntp_fltr_lock);
9769         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
9770                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
9771         if (bit_id < 0) {
9772                 spin_unlock_bh(&bp->ntp_fltr_lock);
9773                 rc = -ENOMEM;
9774                 goto err_free;
9775         }
9776
9777         new_fltr->sw_id = (u16)bit_id;
9778         new_fltr->flow_id = flow_id;
9779         new_fltr->l2_fltr_idx = l2_idx;
9780         new_fltr->rxq = rxq_index;
9781         hlist_add_head_rcu(&new_fltr->hash, head);
9782         bp->ntp_fltr_count++;
9783         spin_unlock_bh(&bp->ntp_fltr_lock);
9784
9785         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
9786         bnxt_queue_sp_work(bp);
9787
9788         return new_fltr->sw_id;
9789
9790 err_free:
9791         kfree(new_fltr);
9792         return rc;
9793 }
9794
9795 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
9796 {
9797         int i;
9798
9799         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
9800                 struct hlist_head *head;
9801                 struct hlist_node *tmp;
9802                 struct bnxt_ntuple_filter *fltr;
9803                 int rc;
9804
9805                 head = &bp->ntp_fltr_hash_tbl[i];
9806                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
9807                         bool del = false;
9808
9809                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
9810                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
9811                                                         fltr->flow_id,
9812                                                         fltr->sw_id)) {
9813                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
9814                                                                          fltr);
9815                                         del = true;
9816                                 }
9817                         } else {
9818                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
9819                                                                        fltr);
9820                                 if (rc)
9821                                         del = true;
9822                                 else
9823                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
9824                         }
9825
9826                         if (del) {
9827                                 spin_lock_bh(&bp->ntp_fltr_lock);
9828                                 hlist_del_rcu(&fltr->hash);
9829                                 bp->ntp_fltr_count--;
9830                                 spin_unlock_bh(&bp->ntp_fltr_lock);
9831                                 synchronize_rcu();
9832                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
9833                                 kfree(fltr);
9834                         }
9835                 }
9836         }
9837         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
9838                 netdev_info(bp->dev, "Receive PF driver unload event!");
9839 }
9840
9841 #else
9842
9843 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
9844 {
9845 }
9846
9847 #endif /* CONFIG_RFS_ACCEL */
9848
9849 static void bnxt_udp_tunnel_add(struct net_device *dev,
9850                                 struct udp_tunnel_info *ti)
9851 {
9852         struct bnxt *bp = netdev_priv(dev);
9853
9854         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
9855                 return;
9856
9857         if (!netif_running(dev))
9858                 return;
9859
9860         switch (ti->type) {
9861         case UDP_TUNNEL_TYPE_VXLAN:
9862                 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port)
9863                         return;
9864
9865                 bp->vxlan_port_cnt++;
9866                 if (bp->vxlan_port_cnt == 1) {
9867                         bp->vxlan_port = ti->port;
9868                         set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
9869                         bnxt_queue_sp_work(bp);
9870                 }
9871                 break;
9872         case UDP_TUNNEL_TYPE_GENEVE:
9873                 if (bp->nge_port_cnt && bp->nge_port != ti->port)
9874                         return;
9875
9876                 bp->nge_port_cnt++;
9877                 if (bp->nge_port_cnt == 1) {
9878                         bp->nge_port = ti->port;
9879                         set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event);
9880                 }
9881                 break;
9882         default:
9883                 return;
9884         }
9885
9886         bnxt_queue_sp_work(bp);
9887 }
9888
9889 static void bnxt_udp_tunnel_del(struct net_device *dev,
9890                                 struct udp_tunnel_info *ti)
9891 {
9892         struct bnxt *bp = netdev_priv(dev);
9893
9894         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
9895                 return;
9896
9897         if (!netif_running(dev))
9898                 return;
9899
9900         switch (ti->type) {
9901         case UDP_TUNNEL_TYPE_VXLAN:
9902                 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port)
9903                         return;
9904                 bp->vxlan_port_cnt--;
9905
9906                 if (bp->vxlan_port_cnt != 0)
9907                         return;
9908
9909                 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
9910                 break;
9911         case UDP_TUNNEL_TYPE_GENEVE:
9912                 if (!bp->nge_port_cnt || bp->nge_port != ti->port)
9913                         return;
9914                 bp->nge_port_cnt--;
9915
9916                 if (bp->nge_port_cnt != 0)
9917                         return;
9918
9919                 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event);
9920                 break;
9921         default:
9922                 return;
9923         }
9924
9925         bnxt_queue_sp_work(bp);
9926 }
9927
9928 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
9929                                struct net_device *dev, u32 filter_mask,
9930                                int nlflags)
9931 {
9932         struct bnxt *bp = netdev_priv(dev);
9933
9934         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
9935                                        nlflags, filter_mask, NULL);
9936 }
9937
9938 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
9939                                u16 flags, struct netlink_ext_ack *extack)
9940 {
9941         struct bnxt *bp = netdev_priv(dev);
9942         struct nlattr *attr, *br_spec;
9943         int rem, rc = 0;
9944
9945         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
9946                 return -EOPNOTSUPP;
9947
9948         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
9949         if (!br_spec)
9950                 return -EINVAL;
9951
9952         nla_for_each_nested(attr, br_spec, rem) {
9953                 u16 mode;
9954
9955                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
9956                         continue;
9957
9958                 if (nla_len(attr) < sizeof(mode))
9959                         return -EINVAL;
9960
9961                 mode = nla_get_u16(attr);
9962                 if (mode == bp->br_mode)
9963                         break;
9964
9965                 rc = bnxt_hwrm_set_br_mode(bp, mode);
9966                 if (!rc)
9967                         bp->br_mode = mode;
9968                 break;
9969         }
9970         return rc;
9971 }
9972
9973 static int bnxt_get_phys_port_name(struct net_device *dev, char *buf,
9974                                    size_t len)
9975 {
9976         struct bnxt *bp = netdev_priv(dev);
9977         int rc;
9978
9979         /* The PF and it's VF-reps only support the switchdev framework */
9980         if (!BNXT_PF(bp))
9981                 return -EOPNOTSUPP;
9982
9983         rc = snprintf(buf, len, "p%d", bp->pf.port_id);
9984
9985         if (rc >= len)
9986                 return -EOPNOTSUPP;
9987         return 0;
9988 }
9989
9990 int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr)
9991 {
9992         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
9993                 return -EOPNOTSUPP;
9994
9995         /* The PF and it's VF-reps only support the switchdev framework */
9996         if (!BNXT_PF(bp))
9997                 return -EOPNOTSUPP;
9998
9999         switch (attr->id) {
10000         case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
10001                 attr->u.ppid.id_len = sizeof(bp->switch_id);
10002                 memcpy(attr->u.ppid.id, bp->switch_id, attr->u.ppid.id_len);
10003                 break;
10004         default:
10005                 return -EOPNOTSUPP;
10006         }
10007         return 0;
10008 }
10009
10010 static int bnxt_swdev_port_attr_get(struct net_device *dev,
10011                                     struct switchdev_attr *attr)
10012 {
10013         return bnxt_port_attr_get(netdev_priv(dev), attr);
10014 }
10015
10016 static const struct switchdev_ops bnxt_switchdev_ops = {
10017         .switchdev_port_attr_get        = bnxt_swdev_port_attr_get
10018 };
10019
10020 static const struct net_device_ops bnxt_netdev_ops = {
10021         .ndo_open               = bnxt_open,
10022         .ndo_start_xmit         = bnxt_start_xmit,
10023         .ndo_stop               = bnxt_close,
10024         .ndo_get_stats64        = bnxt_get_stats64,
10025         .ndo_set_rx_mode        = bnxt_set_rx_mode,
10026         .ndo_do_ioctl           = bnxt_ioctl,
10027         .ndo_validate_addr      = eth_validate_addr,
10028         .ndo_set_mac_address    = bnxt_change_mac_addr,
10029         .ndo_change_mtu         = bnxt_change_mtu,
10030         .ndo_fix_features       = bnxt_fix_features,
10031         .ndo_set_features       = bnxt_set_features,
10032         .ndo_tx_timeout         = bnxt_tx_timeout,
10033 #ifdef CONFIG_BNXT_SRIOV
10034         .ndo_get_vf_config      = bnxt_get_vf_config,
10035         .ndo_set_vf_mac         = bnxt_set_vf_mac,
10036         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
10037         .ndo_set_vf_rate        = bnxt_set_vf_bw,
10038         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
10039         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
10040         .ndo_set_vf_trust       = bnxt_set_vf_trust,
10041 #endif
10042         .ndo_setup_tc           = bnxt_setup_tc,
10043 #ifdef CONFIG_RFS_ACCEL
10044         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
10045 #endif
10046         .ndo_udp_tunnel_add     = bnxt_udp_tunnel_add,
10047         .ndo_udp_tunnel_del     = bnxt_udp_tunnel_del,
10048         .ndo_bpf                = bnxt_xdp,
10049         .ndo_bridge_getlink     = bnxt_bridge_getlink,
10050         .ndo_bridge_setlink     = bnxt_bridge_setlink,
10051         .ndo_get_phys_port_name = bnxt_get_phys_port_name
10052 };
10053
10054 static void bnxt_remove_one(struct pci_dev *pdev)
10055 {
10056         struct net_device *dev = pci_get_drvdata(pdev);
10057         struct bnxt *bp = netdev_priv(dev);
10058
10059         if (BNXT_PF(bp)) {
10060                 bnxt_sriov_disable(bp);
10061                 bnxt_dl_unregister(bp);
10062         }
10063
10064         pci_disable_pcie_error_reporting(pdev);
10065         unregister_netdev(dev);
10066         bnxt_shutdown_tc(bp);
10067         bnxt_cancel_sp_work(bp);
10068         bp->sp_event = 0;
10069
10070         bnxt_clear_int_mode(bp);
10071         bnxt_hwrm_func_drv_unrgtr(bp);
10072         bnxt_free_hwrm_resources(bp);
10073         bnxt_free_hwrm_short_cmd_req(bp);
10074         bnxt_ethtool_free(bp);
10075         bnxt_dcb_free(bp);
10076         kfree(bp->edev);
10077         bp->edev = NULL;
10078         bnxt_free_ctx_mem(bp);
10079         kfree(bp->ctx);
10080         bp->ctx = NULL;
10081         bnxt_cleanup_pci(bp);
10082         bnxt_free_port_stats(bp);
10083         free_netdev(dev);
10084 }
10085
10086 static int bnxt_probe_phy(struct bnxt *bp)
10087 {
10088         int rc = 0;
10089         struct bnxt_link_info *link_info = &bp->link_info;
10090
10091         rc = bnxt_hwrm_phy_qcaps(bp);
10092         if (rc) {
10093                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
10094                            rc);
10095                 return rc;
10096         }
10097         mutex_init(&bp->link_lock);
10098
10099         rc = bnxt_update_link(bp, false);
10100         if (rc) {
10101                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
10102                            rc);
10103                 return rc;
10104         }
10105
10106         /* Older firmware does not have supported_auto_speeds, so assume
10107          * that all supported speeds can be autonegotiated.
10108          */
10109         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
10110                 link_info->support_auto_speeds = link_info->support_speeds;
10111
10112         /*initialize the ethool setting copy with NVM settings */
10113         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
10114                 link_info->autoneg = BNXT_AUTONEG_SPEED;
10115                 if (bp->hwrm_spec_code >= 0x10201) {
10116                         if (link_info->auto_pause_setting &
10117                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
10118                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10119                 } else {
10120                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10121                 }
10122                 link_info->advertising = link_info->auto_link_speeds;
10123         } else {
10124                 link_info->req_link_speed = link_info->force_link_speed;
10125                 link_info->req_duplex = link_info->duplex_setting;
10126         }
10127         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
10128                 link_info->req_flow_ctrl =
10129                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
10130         else
10131                 link_info->req_flow_ctrl = link_info->force_pause_setting;
10132         return rc;
10133 }
10134
10135 static int bnxt_get_max_irq(struct pci_dev *pdev)
10136 {
10137         u16 ctrl;
10138
10139         if (!pdev->msix_cap)
10140                 return 1;
10141
10142         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
10143         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
10144 }
10145
10146 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
10147                                 int *max_cp)
10148 {
10149         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10150         int max_ring_grps = 0, max_irq;
10151
10152         *max_tx = hw_resc->max_tx_rings;
10153         *max_rx = hw_resc->max_rx_rings;
10154         *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
10155         max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
10156                         bnxt_get_ulp_msix_num(bp),
10157                         hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
10158         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
10159                 *max_cp = min_t(int, *max_cp, max_irq);
10160         max_ring_grps = hw_resc->max_hw_ring_grps;
10161         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
10162                 *max_cp -= 1;
10163                 *max_rx -= 2;
10164         }
10165         if (bp->flags & BNXT_FLAG_AGG_RINGS)
10166                 *max_rx >>= 1;
10167         if (bp->flags & BNXT_FLAG_CHIP_P5) {
10168                 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
10169                 /* On P5 chips, max_cp output param should be available NQs */
10170                 *max_cp = max_irq;
10171         }
10172         *max_rx = min_t(int, *max_rx, max_ring_grps);
10173 }
10174
10175 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
10176 {
10177         int rx, tx, cp;
10178
10179         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
10180         *max_rx = rx;
10181         *max_tx = tx;
10182         if (!rx || !tx || !cp)
10183                 return -ENOMEM;
10184
10185         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
10186 }
10187
10188 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
10189                                bool shared)
10190 {
10191         int rc;
10192
10193         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
10194         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
10195                 /* Not enough rings, try disabling agg rings. */
10196                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
10197                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
10198                 if (rc) {
10199                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
10200                         bp->flags |= BNXT_FLAG_AGG_RINGS;
10201                         return rc;
10202                 }
10203                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
10204                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10205                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10206                 bnxt_set_ring_params(bp);
10207         }
10208
10209         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
10210                 int max_cp, max_stat, max_irq;
10211
10212                 /* Reserve minimum resources for RoCE */
10213                 max_cp = bnxt_get_max_func_cp_rings(bp);
10214                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
10215                 max_irq = bnxt_get_max_func_irqs(bp);
10216                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
10217                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
10218                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
10219                         return 0;
10220
10221                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
10222                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
10223                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
10224                 max_cp = min_t(int, max_cp, max_irq);
10225                 max_cp = min_t(int, max_cp, max_stat);
10226                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
10227                 if (rc)
10228                         rc = 0;
10229         }
10230         return rc;
10231 }
10232
10233 /* In initial default shared ring setting, each shared ring must have a
10234  * RX/TX ring pair.
10235  */
10236 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
10237 {
10238         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
10239         bp->rx_nr_rings = bp->cp_nr_rings;
10240         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
10241         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
10242 }
10243
10244 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
10245 {
10246         int dflt_rings, max_rx_rings, max_tx_rings, rc;
10247
10248         if (!bnxt_can_reserve_rings(bp))
10249                 return 0;
10250
10251         if (sh)
10252                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
10253         dflt_rings = netif_get_num_default_rss_queues();
10254         /* Reduce default rings on multi-port cards so that total default
10255          * rings do not exceed CPU count.
10256          */
10257         if (bp->port_count > 1) {
10258                 int max_rings =
10259                         max_t(int, num_online_cpus() / bp->port_count, 1);
10260
10261                 dflt_rings = min_t(int, dflt_rings, max_rings);
10262         }
10263         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
10264         if (rc)
10265                 return rc;
10266         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
10267         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
10268         if (sh)
10269                 bnxt_trim_dflt_sh_rings(bp);
10270         else
10271                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
10272         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
10273
10274         rc = __bnxt_reserve_rings(bp);
10275         if (rc)
10276                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
10277         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10278         if (sh)
10279                 bnxt_trim_dflt_sh_rings(bp);
10280
10281         /* Rings may have been trimmed, re-reserve the trimmed rings. */
10282         if (bnxt_need_reserve_rings(bp)) {
10283                 rc = __bnxt_reserve_rings(bp);
10284                 if (rc)
10285                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
10286                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10287         }
10288         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
10289                 bp->rx_nr_rings++;
10290                 bp->cp_nr_rings++;
10291         }
10292         return rc;
10293 }
10294
10295 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
10296 {
10297         int rc;
10298
10299         if (bp->tx_nr_rings)
10300                 return 0;
10301
10302         bnxt_ulp_irq_stop(bp);
10303         bnxt_clear_int_mode(bp);
10304         rc = bnxt_set_dflt_rings(bp, true);
10305         if (rc) {
10306                 netdev_err(bp->dev, "Not enough rings available.\n");
10307                 goto init_dflt_ring_err;
10308         }
10309         rc = bnxt_init_int_mode(bp);
10310         if (rc)
10311                 goto init_dflt_ring_err;
10312
10313         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10314         if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
10315                 bp->flags |= BNXT_FLAG_RFS;
10316                 bp->dev->features |= NETIF_F_NTUPLE;
10317         }
10318 init_dflt_ring_err:
10319         bnxt_ulp_irq_restart(bp, rc);
10320         return rc;
10321 }
10322
10323 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
10324 {
10325         int rc;
10326
10327         ASSERT_RTNL();
10328         bnxt_hwrm_func_qcaps(bp);
10329
10330         if (netif_running(bp->dev))
10331                 __bnxt_close_nic(bp, true, false);
10332
10333         bnxt_ulp_irq_stop(bp);
10334         bnxt_clear_int_mode(bp);
10335         rc = bnxt_init_int_mode(bp);
10336         bnxt_ulp_irq_restart(bp, rc);
10337
10338         if (netif_running(bp->dev)) {
10339                 if (rc)
10340                         dev_close(bp->dev);
10341                 else
10342                         rc = bnxt_open_nic(bp, true, false);
10343         }
10344
10345         return rc;
10346 }
10347
10348 static int bnxt_init_mac_addr(struct bnxt *bp)
10349 {
10350         int rc = 0;
10351
10352         if (BNXT_PF(bp)) {
10353                 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
10354         } else {
10355 #ifdef CONFIG_BNXT_SRIOV
10356                 struct bnxt_vf_info *vf = &bp->vf;
10357                 bool strict_approval = true;
10358
10359                 if (is_valid_ether_addr(vf->mac_addr)) {
10360                         /* overwrite netdev dev_addr with admin VF MAC */
10361                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
10362                         /* Older PF driver or firmware may not approve this
10363                          * correctly.
10364                          */
10365                         strict_approval = false;
10366                 } else {
10367                         eth_hw_addr_random(bp->dev);
10368                 }
10369                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
10370 #endif
10371         }
10372         return rc;
10373 }
10374
10375 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
10376 {
10377         static int version_printed;
10378         struct net_device *dev;
10379         struct bnxt *bp;
10380         int rc, max_irqs;
10381
10382         if (pci_is_bridge(pdev))
10383                 return -ENODEV;
10384
10385         if (version_printed++ == 0)
10386                 pr_info("%s", version);
10387
10388         max_irqs = bnxt_get_max_irq(pdev);
10389         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
10390         if (!dev)
10391                 return -ENOMEM;
10392
10393         bp = netdev_priv(dev);
10394         bnxt_set_max_func_irqs(bp, max_irqs);
10395
10396         if (bnxt_vf_pciid(ent->driver_data))
10397                 bp->flags |= BNXT_FLAG_VF;
10398
10399         if (pdev->msix_cap)
10400                 bp->flags |= BNXT_FLAG_MSIX_CAP;
10401
10402         rc = bnxt_init_board(pdev, dev);
10403         if (rc < 0)
10404                 goto init_err_free;
10405
10406         dev->netdev_ops = &bnxt_netdev_ops;
10407         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
10408         dev->ethtool_ops = &bnxt_ethtool_ops;
10409         SWITCHDEV_SET_OPS(dev, &bnxt_switchdev_ops);
10410         pci_set_drvdata(pdev, dev);
10411
10412         rc = bnxt_alloc_hwrm_resources(bp);
10413         if (rc)
10414                 goto init_err_pci_clean;
10415
10416         mutex_init(&bp->hwrm_cmd_lock);
10417         rc = bnxt_hwrm_ver_get(bp);
10418         if (rc)
10419                 goto init_err_pci_clean;
10420
10421         if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
10422                 rc = bnxt_alloc_kong_hwrm_resources(bp);
10423                 if (rc)
10424                         bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
10425         }
10426
10427         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
10428             bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
10429                 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
10430                 if (rc)
10431                         goto init_err_pci_clean;
10432         }
10433
10434         if (BNXT_CHIP_P5(bp))
10435                 bp->flags |= BNXT_FLAG_CHIP_P5;
10436
10437         rc = bnxt_hwrm_func_reset(bp);
10438         if (rc)
10439                 goto init_err_pci_clean;
10440
10441         bnxt_hwrm_fw_set_time(bp);
10442
10443         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
10444                            NETIF_F_TSO | NETIF_F_TSO6 |
10445                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
10446                            NETIF_F_GSO_IPXIP4 |
10447                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
10448                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
10449                            NETIF_F_RXCSUM | NETIF_F_GRO;
10450
10451         if (BNXT_SUPPORTS_TPA(bp))
10452                 dev->hw_features |= NETIF_F_LRO;
10453
10454         dev->hw_enc_features =
10455                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
10456                         NETIF_F_TSO | NETIF_F_TSO6 |
10457                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
10458                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
10459                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
10460         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
10461                                     NETIF_F_GSO_GRE_CSUM;
10462         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
10463         dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
10464                             NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
10465         if (BNXT_SUPPORTS_TPA(bp))
10466                 dev->hw_features |= NETIF_F_GRO_HW;
10467         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
10468         if (dev->features & NETIF_F_GRO_HW)
10469                 dev->features &= ~NETIF_F_LRO;
10470         dev->priv_flags |= IFF_UNICAST_FLT;
10471
10472 #ifdef CONFIG_BNXT_SRIOV
10473         init_waitqueue_head(&bp->sriov_cfg_wait);
10474         mutex_init(&bp->sriov_lock);
10475 #endif
10476         if (BNXT_SUPPORTS_TPA(bp)) {
10477                 bp->gro_func = bnxt_gro_func_5730x;
10478                 if (BNXT_CHIP_P4(bp))
10479                         bp->gro_func = bnxt_gro_func_5731x;
10480         }
10481         if (!BNXT_CHIP_P4_PLUS(bp))
10482                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
10483
10484         rc = bnxt_hwrm_func_drv_rgtr(bp);
10485         if (rc)
10486                 goto init_err_pci_clean;
10487
10488         rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0);
10489         if (rc)
10490                 goto init_err_pci_clean;
10491
10492         bp->ulp_probe = bnxt_ulp_probe;
10493
10494         rc = bnxt_hwrm_queue_qportcfg(bp);
10495         if (rc) {
10496                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n",
10497                            rc);
10498                 rc = -1;
10499                 goto init_err_pci_clean;
10500         }
10501         /* Get the MAX capabilities for this function */
10502         rc = bnxt_hwrm_func_qcaps(bp);
10503         if (rc) {
10504                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
10505                            rc);
10506                 rc = -1;
10507                 goto init_err_pci_clean;
10508         }
10509         rc = bnxt_init_mac_addr(bp);
10510         if (rc) {
10511                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
10512                 rc = -EADDRNOTAVAIL;
10513                 goto init_err_pci_clean;
10514         }
10515
10516         bnxt_hwrm_func_qcfg(bp);
10517         bnxt_hwrm_vnic_qcaps(bp);
10518         bnxt_hwrm_port_led_qcaps(bp);
10519         bnxt_ethtool_init(bp);
10520         bnxt_dcb_init(bp);
10521
10522         /* MTU range: 60 - FW defined max */
10523         dev->min_mtu = ETH_ZLEN;
10524         dev->max_mtu = bp->max_mtu;
10525
10526         rc = bnxt_probe_phy(bp);
10527         if (rc)
10528                 goto init_err_pci_clean;
10529
10530         bnxt_set_rx_skb_mode(bp, false);
10531         bnxt_set_tpa_flags(bp);
10532         bnxt_set_ring_params(bp);
10533         rc = bnxt_set_dflt_rings(bp, true);
10534         if (rc) {
10535                 netdev_err(bp->dev, "Not enough rings available.\n");
10536                 rc = -ENOMEM;
10537                 goto init_err_pci_clean;
10538         }
10539
10540         /* Default RSS hash cfg. */
10541         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
10542                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
10543                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
10544                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
10545         if (BNXT_CHIP_P4(bp) && bp->hwrm_spec_code >= 0x10501) {
10546                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
10547                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
10548                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
10549         }
10550
10551         if (bnxt_rfs_supported(bp)) {
10552                 dev->hw_features |= NETIF_F_NTUPLE;
10553                 if (bnxt_rfs_capable(bp)) {
10554                         bp->flags |= BNXT_FLAG_RFS;
10555                         dev->features |= NETIF_F_NTUPLE;
10556                 }
10557         }
10558
10559         if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
10560                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
10561
10562         rc = bnxt_init_int_mode(bp);
10563         if (rc)
10564                 goto init_err_pci_clean;
10565
10566         /* No TC has been set yet and rings may have been trimmed due to
10567          * limited MSIX, so we re-initialize the TX rings per TC.
10568          */
10569         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10570
10571         bnxt_get_wol_settings(bp);
10572         if (bp->flags & BNXT_FLAG_WOL_CAP)
10573                 device_set_wakeup_enable(&pdev->dev, bp->wol);
10574         else
10575                 device_set_wakeup_capable(&pdev->dev, false);
10576
10577         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
10578
10579         bnxt_hwrm_coal_params_qcaps(bp);
10580
10581         if (BNXT_PF(bp)) {
10582                 if (!bnxt_pf_wq) {
10583                         bnxt_pf_wq =
10584                                 create_singlethread_workqueue("bnxt_pf_wq");
10585                         if (!bnxt_pf_wq) {
10586                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
10587                                 goto init_err_pci_clean;
10588                         }
10589                 }
10590                 bnxt_init_tc(bp);
10591         }
10592
10593         rc = register_netdev(dev);
10594         if (rc)
10595                 goto init_err_cleanup_tc;
10596
10597         if (BNXT_PF(bp))
10598                 bnxt_dl_register(bp);
10599
10600         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
10601                     board_info[ent->driver_data].name,
10602                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
10603         pcie_print_link_status(pdev);
10604
10605         return 0;
10606
10607 init_err_cleanup_tc:
10608         bnxt_shutdown_tc(bp);
10609         bnxt_clear_int_mode(bp);
10610
10611 init_err_pci_clean:
10612         bnxt_free_hwrm_resources(bp);
10613         bnxt_free_ctx_mem(bp);
10614         kfree(bp->ctx);
10615         bp->ctx = NULL;
10616         bnxt_cleanup_pci(bp);
10617
10618 init_err_free:
10619         free_netdev(dev);
10620         return rc;
10621 }
10622
10623 static void bnxt_shutdown(struct pci_dev *pdev)
10624 {
10625         struct net_device *dev = pci_get_drvdata(pdev);
10626         struct bnxt *bp;
10627
10628         if (!dev)
10629                 return;
10630
10631         rtnl_lock();
10632         bp = netdev_priv(dev);
10633         if (!bp)
10634                 goto shutdown_exit;
10635
10636         if (netif_running(dev))
10637                 dev_close(dev);
10638
10639         bnxt_ulp_shutdown(bp);
10640
10641         if (system_state == SYSTEM_POWER_OFF) {
10642                 bnxt_clear_int_mode(bp);
10643                 pci_wake_from_d3(pdev, bp->wol);
10644                 pci_set_power_state(pdev, PCI_D3hot);
10645         }
10646
10647 shutdown_exit:
10648         rtnl_unlock();
10649 }
10650
10651 #ifdef CONFIG_PM_SLEEP
10652 static int bnxt_suspend(struct device *device)
10653 {
10654         struct pci_dev *pdev = to_pci_dev(device);
10655         struct net_device *dev = pci_get_drvdata(pdev);
10656         struct bnxt *bp = netdev_priv(dev);
10657         int rc = 0;
10658
10659         rtnl_lock();
10660         if (netif_running(dev)) {
10661                 netif_device_detach(dev);
10662                 rc = bnxt_close(dev);
10663         }
10664         bnxt_hwrm_func_drv_unrgtr(bp);
10665         rtnl_unlock();
10666         return rc;
10667 }
10668
10669 static int bnxt_resume(struct device *device)
10670 {
10671         struct pci_dev *pdev = to_pci_dev(device);
10672         struct net_device *dev = pci_get_drvdata(pdev);
10673         struct bnxt *bp = netdev_priv(dev);
10674         int rc = 0;
10675
10676         rtnl_lock();
10677         if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) {
10678                 rc = -ENODEV;
10679                 goto resume_exit;
10680         }
10681         rc = bnxt_hwrm_func_reset(bp);
10682         if (rc) {
10683                 rc = -EBUSY;
10684                 goto resume_exit;
10685         }
10686         bnxt_get_wol_settings(bp);
10687         if (netif_running(dev)) {
10688                 rc = bnxt_open(dev);
10689                 if (!rc)
10690                         netif_device_attach(dev);
10691         }
10692
10693 resume_exit:
10694         rtnl_unlock();
10695         return rc;
10696 }
10697
10698 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
10699 #define BNXT_PM_OPS (&bnxt_pm_ops)
10700
10701 #else
10702
10703 #define BNXT_PM_OPS NULL
10704
10705 #endif /* CONFIG_PM_SLEEP */
10706
10707 /**
10708  * bnxt_io_error_detected - called when PCI error is detected
10709  * @pdev: Pointer to PCI device
10710  * @state: The current pci connection state
10711  *
10712  * This function is called after a PCI bus error affecting
10713  * this device has been detected.
10714  */
10715 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
10716                                                pci_channel_state_t state)
10717 {
10718         struct net_device *netdev = pci_get_drvdata(pdev);
10719         struct bnxt *bp = netdev_priv(netdev);
10720
10721         netdev_info(netdev, "PCI I/O error detected\n");
10722
10723         rtnl_lock();
10724         netif_device_detach(netdev);
10725
10726         bnxt_ulp_stop(bp);
10727
10728         if (state == pci_channel_io_perm_failure) {
10729                 rtnl_unlock();
10730                 return PCI_ERS_RESULT_DISCONNECT;
10731         }
10732
10733         if (netif_running(netdev))
10734                 bnxt_close(netdev);
10735
10736         pci_disable_device(pdev);
10737         rtnl_unlock();
10738
10739         /* Request a slot slot reset. */
10740         return PCI_ERS_RESULT_NEED_RESET;
10741 }
10742
10743 /**
10744  * bnxt_io_slot_reset - called after the pci bus has been reset.
10745  * @pdev: Pointer to PCI device
10746  *
10747  * Restart the card from scratch, as if from a cold-boot.
10748  * At this point, the card has exprienced a hard reset,
10749  * followed by fixups by BIOS, and has its config space
10750  * set up identically to what it was at cold boot.
10751  */
10752 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
10753 {
10754         struct net_device *netdev = pci_get_drvdata(pdev);
10755         struct bnxt *bp = netdev_priv(netdev);
10756         int err = 0;
10757         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
10758
10759         netdev_info(bp->dev, "PCI Slot Reset\n");
10760
10761         rtnl_lock();
10762
10763         if (pci_enable_device(pdev)) {
10764                 dev_err(&pdev->dev,
10765                         "Cannot re-enable PCI device after reset.\n");
10766         } else {
10767                 pci_set_master(pdev);
10768
10769                 err = bnxt_hwrm_func_reset(bp);
10770                 if (!err && netif_running(netdev))
10771                         err = bnxt_open(netdev);
10772
10773                 if (!err) {
10774                         result = PCI_ERS_RESULT_RECOVERED;
10775                         bnxt_ulp_start(bp);
10776                 }
10777         }
10778
10779         if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
10780                 dev_close(netdev);
10781
10782         rtnl_unlock();
10783
10784         return PCI_ERS_RESULT_RECOVERED;
10785 }
10786
10787 /**
10788  * bnxt_io_resume - called when traffic can start flowing again.
10789  * @pdev: Pointer to PCI device
10790  *
10791  * This callback is called when the error recovery driver tells
10792  * us that its OK to resume normal operation.
10793  */
10794 static void bnxt_io_resume(struct pci_dev *pdev)
10795 {
10796         struct net_device *netdev = pci_get_drvdata(pdev);
10797
10798         rtnl_lock();
10799
10800         netif_device_attach(netdev);
10801
10802         rtnl_unlock();
10803 }
10804
10805 static const struct pci_error_handlers bnxt_err_handler = {
10806         .error_detected = bnxt_io_error_detected,
10807         .slot_reset     = bnxt_io_slot_reset,
10808         .resume         = bnxt_io_resume
10809 };
10810
10811 static struct pci_driver bnxt_pci_driver = {
10812         .name           = DRV_MODULE_NAME,
10813         .id_table       = bnxt_pci_tbl,
10814         .probe          = bnxt_init_one,
10815         .remove         = bnxt_remove_one,
10816         .shutdown       = bnxt_shutdown,
10817         .driver.pm      = BNXT_PM_OPS,
10818         .err_handler    = &bnxt_err_handler,
10819 #if defined(CONFIG_BNXT_SRIOV)
10820         .sriov_configure = bnxt_sriov_configure,
10821 #endif
10822 };
10823
10824 static int __init bnxt_init(void)
10825 {
10826         bnxt_debug_init();
10827         return pci_register_driver(&bnxt_pci_driver);
10828 }
10829
10830 static void __exit bnxt_exit(void)
10831 {
10832         pci_unregister_driver(&bnxt_pci_driver);
10833         if (bnxt_pf_wq)
10834                 destroy_workqueue(bnxt_pf_wq);
10835         bnxt_debug_exit();
10836 }
10837
10838 module_init(bnxt_init);
10839 module_exit(bnxt_exit);