drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  30 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  31 MODULE_AUTHOR("Emulex Corporation");
  32 MODULE_LICENSE("GPL");
  33
  34 static unsigned int num_vfs;
  35 module_param(num_vfs, uint, S_IRUGO);
  36 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  37
  38 static ushort rx_frag_size = 2048;
  39 module_param(rx_frag_size, ushort, S_IRUGO);
  40 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  41
  42 static const struct pci_device_id be_dev_ids[] = {
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  44         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  46         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  50         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  51         { 0 }
  52 };
  53 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  54 /* UE Status Low CSR */
  55 static const char * const ue_status_low_desc[] = {
  56         "CEV",
  57         "CTX",
  58         "DBUF",
  59         "ERX",
  60         "Host",
  61         "MPU",
  62         "NDMA",
  63         "PTC ",
  64         "RDMA ",
  65         "RXF ",
  66         "RXIPS ",
  67         "RXULP0 ",
  68         "RXULP1 ",
  69         "RXULP2 ",
  70         "TIM ",
  71         "TPOST ",
  72         "TPRE ",
  73         "TXIPS ",
  74         "TXULP0 ",
  75         "TXULP1 ",
  76         "UC ",
  77         "WDMA ",
  78         "TXULP2 ",
  79         "HOST1 ",
  80         "P0_OB_LINK ",
  81         "P1_OB_LINK ",
  82         "HOST_GPIO ",
  83         "MBOX ",
  84         "ERX2 ",
  85         "SPARE ",
  86         "JTAG ",
  87         "MPU_INTPEND "
  88 };
  89
  90 /* UE Status High CSR */
  91 static const char * const ue_status_hi_desc[] = {
  92         "LPCMEMHOST",
  93         "MGMT_MAC",
  94         "PCS0ONLINE",
  95         "MPU_IRAM",
  96         "PCS1ONLINE",
  97         "PCTL0",
  98         "PCTL1",
  99         "PMEM",
 100         "RR",
 101         "TXPB",
 102         "RXPP",
 103         "XAUI",
 104         "TXP",
 105         "ARM",
 106         "IPC",
 107         "HOST2",
 108         "HOST3",
 109         "HOST4",
 110         "HOST5",
 111         "HOST6",
 112         "HOST7",
 113         "ECRC",
 114         "Poison TLP",
 115         "NETC",
 116         "PERIPH",
 117         "LLTXULP",
 118         "D2P",
 119         "RCON",
 120         "LDMA",
 121         "LLTXP",
 122         "LLTXPB",
 123         "Unknown"
 124 };
 125
 126 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 127 {
 128         struct be_dma_mem *mem = &q->dma_mem;
 129
 130         if (mem->va) {
 131                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 132                                   mem->dma);
 133                 mem->va = NULL;
 134         }
 135 }
 136
 137 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 138                           u16 len, u16 entry_size)
 139 {
 140         struct be_dma_mem *mem = &q->dma_mem;
 141
 142         memset(q, 0, sizeof(*q));
 143         q->len = len;
 144         q->entry_size = entry_size;
 145         mem->size = len * entry_size;
 146         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 147                                       GFP_KERNEL);
 148         if (!mem->va)
 149                 return -ENOMEM;
 150         return 0;
 151 }
 152
 153 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 154 {
 155         u32 reg, enabled;
 156
 157         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 158                               &reg);
 159         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 160
 161         if (!enabled && enable)
 162                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 163         else if (enabled && !enable)
 164                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 165         else
 166                 return;
 167
 168         pci_write_config_dword(adapter->pdev,
 169                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 170 }
 171
 172 static void be_intr_set(struct be_adapter *adapter, bool enable)
 173 {
 174         int status = 0;
 175
 176         /* On lancer interrupts can't be controlled via this register */
 177         if (lancer_chip(adapter))
 178                 return;
 179
 180         if (adapter->eeh_error)
 181                 return;
 182
 183         status = be_cmd_intr_set(adapter, enable);
 184         if (status)
 185                 be_reg_intr_set(adapter, enable);
 186 }
 187
 188 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 189 {
 190         u32 val = 0;
 191
 192         val |= qid & DB_RQ_RING_ID_MASK;
 193         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 194
 195         wmb();
 196         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 197 }
 198
 199 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 200                           u16 posted)
 201 {
 202         u32 val = 0;
 203
 204         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 205         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 206
 207         wmb();
 208         iowrite32(val, adapter->db + txo->db_offset);
 209 }
 210
 211 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 212                          bool arm, bool clear_int, u16 num_popped)
 213 {
 214         u32 val = 0;
 215
 216         val |= qid & DB_EQ_RING_ID_MASK;
 217         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 218
 219         if (adapter->eeh_error)
 220                 return;
 221
 222         if (arm)
 223                 val |= 1 << DB_EQ_REARM_SHIFT;
 224         if (clear_int)
 225                 val |= 1 << DB_EQ_CLR_SHIFT;
 226         val |= 1 << DB_EQ_EVNT_SHIFT;
 227         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 228         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 229 }
 230
 231 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 232 {
 233         u32 val = 0;
 234
 235         val |= qid & DB_CQ_RING_ID_MASK;
 236         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 237                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 238
 239         if (adapter->eeh_error)
 240                 return;
 241
 242         if (arm)
 243                 val |= 1 << DB_CQ_REARM_SHIFT;
 244         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 245         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 246 }
 247
 248 static int be_mac_addr_set(struct net_device *netdev, void *p)
 249 {
 250         struct be_adapter *adapter = netdev_priv(netdev);
 251         struct device *dev = &adapter->pdev->dev;
 252         struct sockaddr *addr = p;
 253         int status;
 254         u8 mac[ETH_ALEN];
 255         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 256
 257         if (!is_valid_ether_addr(addr->sa_data))
 258                 return -EADDRNOTAVAIL;
 259
 260         /* Proceed further only if, User provided MAC is different
 261          * from active MAC
 262          */
 263         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 264                 return 0;
 265
 266         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 267          * privilege or if PF did not provision the new MAC address.
 268          * On BE3, this cmd will always fail if the VF doesn't have the
 269          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 270          * the MAC for the VF.
 271          */
 272         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 273                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 274         if (!status) {
 275                 curr_pmac_id = adapter->pmac_id[0];
 276
 277                 /* Delete the old programmed MAC. This call may fail if the
 278                  * old MAC was already deleted by the PF driver.
 279                  */
 280                 if (adapter->pmac_id[0] != old_pmac_id)
 281                         be_cmd_pmac_del(adapter, adapter->if_handle,
 282                                         old_pmac_id, 0);
 283         }
 284
 285         /* Decide if the new MAC is successfully activated only after
 286          * querying the FW
 287          */
 288         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 289                                        adapter->if_handle, true, 0);
 290         if (status)
 291                 goto err;
 292
 293         /* The MAC change did not happen, either due to lack of privilege
 294          * or PF didn't pre-provision.
 295          */
 296         if (!ether_addr_equal(addr->sa_data, mac)) {
 297                 status = -EPERM;
 298                 goto err;
 299         }
 300
 301         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 302         dev_info(dev, "MAC address changed to %pM\n", mac);
 303         return 0;
 304 err:
 305         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 306         return status;
 307 }
 308
 309 /* BE2 supports only v0 cmd */
 310 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 311 {
 312         if (BE2_chip(adapter)) {
 313                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 314
 315                 return &cmd->hw_stats;
 316         } else if (BE3_chip(adapter)) {
 317                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 318
 319                 return &cmd->hw_stats;
 320         } else {
 321                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 322
 323                 return &cmd->hw_stats;
 324         }
 325 }
 326
 327 /* BE2 supports only v0 cmd */
 328 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 329 {
 330         if (BE2_chip(adapter)) {
 331                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 332
 333                 return &hw_stats->erx;
 334         } else if (BE3_chip(adapter)) {
 335                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 336
 337                 return &hw_stats->erx;
 338         } else {
 339                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 340
 341                 return &hw_stats->erx;
 342         }
 343 }
 344
 345 static void populate_be_v0_stats(struct be_adapter *adapter)
 346 {
 347         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 348         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 349         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 350         struct be_port_rxf_stats_v0 *port_stats =
 351                                         &rxf_stats->port[adapter->port_num];
 352         struct be_drv_stats *drvs = &adapter->drv_stats;
 353
 354         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 355         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 356         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 357         drvs->rx_control_frames = port_stats->rx_control_frames;
 358         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 359         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 360         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 361         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 362         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 363         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 364         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 365         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 366         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 367         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 368         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 369         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 370         drvs->rx_dropped_header_too_small =
 371                 port_stats->rx_dropped_header_too_small;
 372         drvs->rx_address_filtered =
 373                                         port_stats->rx_address_filtered +
 374                                         port_stats->rx_vlan_filtered;
 375         drvs->rx_alignment_symbol_errors =
 376                 port_stats->rx_alignment_symbol_errors;
 377
 378         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 379         drvs->tx_controlframes = port_stats->tx_controlframes;
 380
 381         if (adapter->port_num)
 382                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 383         else
 384                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 385         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 386         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 387         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 388         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 389         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 390         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 391         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 392 }
 393
 394 static void populate_be_v1_stats(struct be_adapter *adapter)
 395 {
 396         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 397         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 398         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 399         struct be_port_rxf_stats_v1 *port_stats =
 400                                         &rxf_stats->port[adapter->port_num];
 401         struct be_drv_stats *drvs = &adapter->drv_stats;
 402
 403         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 404         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 405         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 406         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 407         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 408         drvs->rx_control_frames = port_stats->rx_control_frames;
 409         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 410         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 411         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 412         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 413         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 414         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 415         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 416         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 417         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 418         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 419         drvs->rx_dropped_header_too_small =
 420                 port_stats->rx_dropped_header_too_small;
 421         drvs->rx_input_fifo_overflow_drop =
 422                 port_stats->rx_input_fifo_overflow_drop;
 423         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 424         drvs->rx_alignment_symbol_errors =
 425                 port_stats->rx_alignment_symbol_errors;
 426         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 427         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 428         drvs->tx_controlframes = port_stats->tx_controlframes;
 429         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 430         drvs->jabber_events = port_stats->jabber_events;
 431         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 432         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 433         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 434         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 435         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 436         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 437         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 438 }
 439
 440 static void populate_be_v2_stats(struct be_adapter *adapter)
 441 {
 442         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 443         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 444         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 445         struct be_port_rxf_stats_v2 *port_stats =
 446                                         &rxf_stats->port[adapter->port_num];
 447         struct be_drv_stats *drvs = &adapter->drv_stats;
 448
 449         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 450         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 451         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 452         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 453         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 454         drvs->rx_control_frames = port_stats->rx_control_frames;
 455         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 456         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 457         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 458         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 459         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 460         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 461         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 462         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 463         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 464         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 465         drvs->rx_dropped_header_too_small =
 466                 port_stats->rx_dropped_header_too_small;
 467         drvs->rx_input_fifo_overflow_drop =
 468                 port_stats->rx_input_fifo_overflow_drop;
 469         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 470         drvs->rx_alignment_symbol_errors =
 471                 port_stats->rx_alignment_symbol_errors;
 472         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 473         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 474         drvs->tx_controlframes = port_stats->tx_controlframes;
 475         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 476         drvs->jabber_events = port_stats->jabber_events;
 477         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 478         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 479         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 480         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 481         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 482         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 483         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 484         if (be_roce_supported(adapter)) {
 485                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 486                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 487                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 488                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 489                 drvs->roce_drops_payload_len =
 490                         port_stats->roce_drops_payload_len;
 491         }
 492 }
 493
 494 static void populate_lancer_stats(struct be_adapter *adapter)
 495 {
 496         struct be_drv_stats *drvs = &adapter->drv_stats;
 497         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 498
 499         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 500         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 501         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 502         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 503         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 504         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 505         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 506         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 507         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 508         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 509         drvs->rx_dropped_tcp_length =
 510                                 pport_stats->rx_dropped_invalid_tcp_length;
 511         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 512         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 513         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 514         drvs->rx_dropped_header_too_small =
 515                                 pport_stats->rx_dropped_header_too_small;
 516         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 517         drvs->rx_address_filtered =
 518                                         pport_stats->rx_address_filtered +
 519                                         pport_stats->rx_vlan_filtered;
 520         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 521         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 522         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 523         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 524         drvs->jabber_events = pport_stats->rx_jabbers;
 525         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 526         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 527         drvs->rx_drops_too_many_frags =
 528                                 pport_stats->rx_drops_too_many_frags_lo;
 529 }
 530
 531 static void accumulate_16bit_val(u32 *acc, u16 val)
 532 {
 533 #define lo(x)                   (x & 0xFFFF)
 534 #define hi(x)                   (x & 0xFFFF0000)
 535         bool wrapped = val < lo(*acc);
 536         u32 newacc = hi(*acc) + val;
 537
 538         if (wrapped)
 539                 newacc += 65536;
 540         ACCESS_ONCE(*acc) = newacc;
 541 }
 542
 543 static void populate_erx_stats(struct be_adapter *adapter,
 544                                struct be_rx_obj *rxo, u32 erx_stat)
 545 {
 546         if (!BEx_chip(adapter))
 547                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 548         else
 549                 /* below erx HW counter can actually wrap around after
 550                  * 65535. Driver accumulates a 32-bit value
 551                  */
 552                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 553                                      (u16)erx_stat);
 554 }
 555
 556 void be_parse_stats(struct be_adapter *adapter)
 557 {
 558         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 559         struct be_rx_obj *rxo;
 560         int i;
 561         u32 erx_stat;
 562
 563         if (lancer_chip(adapter)) {
 564                 populate_lancer_stats(adapter);
 565         } else {
 566                 if (BE2_chip(adapter))
 567                         populate_be_v0_stats(adapter);
 568                 else if (BE3_chip(adapter))
 569                         /* for BE3 */
 570                         populate_be_v1_stats(adapter);
 571                 else
 572                         populate_be_v2_stats(adapter);
 573
 574                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 575                 for_all_rx_queues(adapter, rxo, i) {
 576                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 577                         populate_erx_stats(adapter, rxo, erx_stat);
 578                 }
 579         }
 580 }
 581
 582 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 583                                                 struct rtnl_link_stats64 *stats)
 584 {
 585         struct be_adapter *adapter = netdev_priv(netdev);
 586         struct be_drv_stats *drvs = &adapter->drv_stats;
 587         struct be_rx_obj *rxo;
 588         struct be_tx_obj *txo;
 589         u64 pkts, bytes;
 590         unsigned int start;
 591         int i;
 592
 593         for_all_rx_queues(adapter, rxo, i) {
 594                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 595
 596                 do {
 597                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 598                         pkts = rx_stats(rxo)->rx_pkts;
 599                         bytes = rx_stats(rxo)->rx_bytes;
 600                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 601                 stats->rx_packets += pkts;
 602                 stats->rx_bytes += bytes;
 603                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 604                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 605                                         rx_stats(rxo)->rx_drops_no_frags;
 606         }
 607
 608         for_all_tx_queues(adapter, txo, i) {
 609                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 610
 611                 do {
 612                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 613                         pkts = tx_stats(txo)->tx_pkts;
 614                         bytes = tx_stats(txo)->tx_bytes;
 615                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 616                 stats->tx_packets += pkts;
 617                 stats->tx_bytes += bytes;
 618         }
 619
 620         /* bad pkts received */
 621         stats->rx_errors = drvs->rx_crc_errors +
 622                 drvs->rx_alignment_symbol_errors +
 623                 drvs->rx_in_range_errors +
 624                 drvs->rx_out_range_errors +
 625                 drvs->rx_frame_too_long +
 626                 drvs->rx_dropped_too_small +
 627                 drvs->rx_dropped_too_short +
 628                 drvs->rx_dropped_header_too_small +
 629                 drvs->rx_dropped_tcp_length +
 630                 drvs->rx_dropped_runt;
 631
 632         /* detailed rx errors */
 633         stats->rx_length_errors = drvs->rx_in_range_errors +
 634                 drvs->rx_out_range_errors +
 635                 drvs->rx_frame_too_long;
 636
 637         stats->rx_crc_errors = drvs->rx_crc_errors;
 638
 639         /* frame alignment errors */
 640         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 641
 642         /* receiver fifo overrun */
 643         /* drops_no_pbuf is no per i/f, it's per BE card */
 644         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 645                                 drvs->rx_input_fifo_overflow_drop +
 646                                 drvs->rx_drops_no_pbuf;
 647         return stats;
 648 }
 649
 650 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 651 {
 652         struct net_device *netdev = adapter->netdev;
 653
 654         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 655                 netif_carrier_off(netdev);
 656                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 657         }
 658
 659         if (link_status)
 660                 netif_carrier_on(netdev);
 661         else
 662                 netif_carrier_off(netdev);
 663 }
 664
 665 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)
 666 {
 667         struct be_tx_stats *stats = tx_stats(txo);
 668
 669         u64_stats_update_begin(&stats->sync);
 670         stats->tx_reqs++;
 671         stats->tx_bytes += skb->len;
 672         stats->tx_pkts += (skb_shinfo(skb)->gso_segs ? : 1);
 673         u64_stats_update_end(&stats->sync);
 674 }
 675
 676 /* Returns number of WRBs needed for the skb */
 677 static u32 skb_wrb_cnt(struct sk_buff *skb)
 678 {
 679         /* +1 for the header wrb */
 680         return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags;
 681 }
 682
 683 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 684 {
 685         wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr));
 686         wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr));
 687         wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK);
 688         wrb->rsvd0 = 0;
 689 }
 690
 691 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb
 692  * to avoid the swap and shift/mask operations in wrb_fill().
 693  */
 694 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb)
 695 {
 696         wrb->frag_pa_hi = 0;
 697         wrb->frag_pa_lo = 0;
 698         wrb->frag_len = 0;
 699         wrb->rsvd0 = 0;
 700 }
 701
 702 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 703                                      struct sk_buff *skb)
 704 {
 705         u8 vlan_prio;
 706         u16 vlan_tag;
 707
 708         vlan_tag = skb_vlan_tag_get(skb);
 709         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 710         /* If vlan priority provided by OS is NOT in available bmap */
 711         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 712                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 713                                 adapter->recommended_prio;
 714
 715         return vlan_tag;
 716 }
 717
 718 /* Used only for IP tunnel packets */
 719 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 720 {
 721         return (inner_ip_hdr(skb)->version == 4) ?
 722                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 723 }
 724
 725 static u16 skb_ip_proto(struct sk_buff *skb)
 726 {
 727         return (ip_hdr(skb)->version == 4) ?
 728                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 729 }
 730
 731 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 732                          struct sk_buff *skb, u32 wrb_cnt, u32 len,
 733                          bool skip_hw_vlan)
 734 {
 735         u16 vlan_tag, proto;
 736
 737         memset(hdr, 0, sizeof(*hdr));
 738
 739         SET_TX_WRB_HDR_BITS(crc, hdr, 1);
 740
 741         if (skb_is_gso(skb)) {
 742                 SET_TX_WRB_HDR_BITS(lso, hdr, 1);
 743                 SET_TX_WRB_HDR_BITS(lso_mss, hdr, skb_shinfo(skb)->gso_size);
 744                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 745                         SET_TX_WRB_HDR_BITS(lso6, hdr, 1);
 746         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 747                 if (skb->encapsulation) {
 748                         SET_TX_WRB_HDR_BITS(ipcs, hdr, 1);
 749                         proto = skb_inner_ip_proto(skb);
 750                 } else {
 751                         proto = skb_ip_proto(skb);
 752                 }
 753                 if (proto == IPPROTO_TCP)
 754                         SET_TX_WRB_HDR_BITS(tcpcs, hdr, 1);
 755                 else if (proto == IPPROTO_UDP)
 756                         SET_TX_WRB_HDR_BITS(udpcs, hdr, 1);
 757         }
 758
 759         if (skb_vlan_tag_present(skb)) {
 760                 SET_TX_WRB_HDR_BITS(vlan, hdr, 1);
 761                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 762                 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, vlan_tag);
 763         }
 764
 765         SET_TX_WRB_HDR_BITS(num_wrb, hdr, wrb_cnt);
 766         SET_TX_WRB_HDR_BITS(len, hdr, len);
 767
 768         /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0
 769          * When this hack is not needed, the evt bit is set while ringing DB
 770          */
 771         if (skip_hw_vlan)
 772                 SET_TX_WRB_HDR_BITS(event, hdr, 1);
 773 }
 774
 775 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 776                           bool unmap_single)
 777 {
 778         dma_addr_t dma;
 779         u32 frag_len = le32_to_cpu(wrb->frag_len);
 780
 781
 782         dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 |
 783                 (u64)le32_to_cpu(wrb->frag_pa_lo);
 784         if (frag_len) {
 785                 if (unmap_single)
 786                         dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE);
 787                 else
 788                         dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE);
 789         }
 790 }
 791
 792 /* Returns the number of WRBs used up by the skb */
 793 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,
 794                            struct sk_buff *skb, bool skip_hw_vlan)
 795 {
 796         u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb);
 797         struct device *dev = &adapter->pdev->dev;
 798         struct be_queue_info *txq = &txo->q;
 799         struct be_eth_hdr_wrb *hdr;
 800         bool map_single = false;
 801         struct be_eth_wrb *wrb;
 802         dma_addr_t busaddr;
 803         u16 head = txq->head;
 804
 805         hdr = queue_head_node(txq);
 806         wrb_fill_hdr(adapter, hdr, skb, wrb_cnt, skb->len, skip_hw_vlan);
 807         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 808
 809         queue_head_inc(txq);
 810
 811         if (skb->len > skb->data_len) {
 812                 int len = skb_headlen(skb);
 813
 814                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 815                 if (dma_mapping_error(dev, busaddr))
 816                         goto dma_err;
 817                 map_single = true;
 818                 wrb = queue_head_node(txq);
 819                 wrb_fill(wrb, busaddr, len);
 820                 queue_head_inc(txq);
 821                 copied += len;
 822         }
 823
 824         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 825                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 826
 827                 busaddr = skb_frag_dma_map(dev, frag, 0,
 828                                            skb_frag_size(frag), DMA_TO_DEVICE);
 829                 if (dma_mapping_error(dev, busaddr))
 830                         goto dma_err;
 831                 wrb = queue_head_node(txq);
 832                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 833                 queue_head_inc(txq);
 834                 copied += skb_frag_size(frag);
 835         }
 836
 837         BUG_ON(txo->sent_skb_list[head]);
 838         txo->sent_skb_list[head] = skb;
 839         txo->last_req_hdr = head;
 840         atomic_add(wrb_cnt, &txq->used);
 841         txo->last_req_wrb_cnt = wrb_cnt;
 842         txo->pend_wrb_cnt += wrb_cnt;
 843
 844         be_tx_stats_update(txo, skb);
 845         return wrb_cnt;
 846
 847 dma_err:
 848         /* Bring the queue back to the state it was in before this
 849          * routine was invoked.
 850          */
 851         txq->head = head;
 852         /* skip the first wrb (hdr); it's not mapped */
 853         queue_head_inc(txq);
 854         while (copied) {
 855                 wrb = queue_head_node(txq);
 856                 unmap_tx_frag(dev, wrb, map_single);
 857                 map_single = false;
 858                 copied -= le32_to_cpu(wrb->frag_len);
 859                 adapter->drv_stats.dma_map_errors++;
 860                 queue_head_inc(txq);
 861         }
 862         txq->head = head;
 863         return 0;
 864 }
 865
 866 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter)
 867 {
 868         return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD;
 869 }
 870
 871 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 872                                              struct sk_buff *skb,
 873                                              bool *skip_hw_vlan)
 874 {
 875         u16 vlan_tag = 0;
 876
 877         skb = skb_share_check(skb, GFP_ATOMIC);
 878         if (unlikely(!skb))
 879                 return skb;
 880
 881         if (skb_vlan_tag_present(skb))
 882                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 883
 884         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 885                 if (!vlan_tag)
 886                         vlan_tag = adapter->pvid;
 887                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 888                  * skip VLAN insertion
 889                  */
 890                 if (skip_hw_vlan)
 891                         *skip_hw_vlan = true;
 892         }
 893
 894         if (vlan_tag) {
 895                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 896                                                 vlan_tag);
 897                 if (unlikely(!skb))
 898                         return skb;
 899                 skb->vlan_tci = 0;
 900         }
 901
 902         /* Insert the outer VLAN, if any */
 903         if (adapter->qnq_vid) {
 904                 vlan_tag = adapter->qnq_vid;
 905                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 906                                                 vlan_tag);
 907                 if (unlikely(!skb))
 908                         return skb;
 909                 if (skip_hw_vlan)
 910                         *skip_hw_vlan = true;
 911         }
 912
 913         return skb;
 914 }
 915
 916 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 917 {
 918         struct ethhdr *eh = (struct ethhdr *)skb->data;
 919         u16 offset = ETH_HLEN;
 920
 921         if (eh->h_proto == htons(ETH_P_IPV6)) {
 922                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 923
 924                 offset += sizeof(struct ipv6hdr);
 925                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 926                     ip6h->nexthdr != NEXTHDR_UDP) {
 927                         struct ipv6_opt_hdr *ehdr =
 928                                 (struct ipv6_opt_hdr *)(skb->data + offset);
 929
 930                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 931                         if (ehdr->hdrlen == 0xff)
 932                                 return true;
 933                 }
 934         }
 935         return false;
 936 }
 937
 938 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 939 {
 940         return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 941 }
 942
 943 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
 944 {
 945         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 946 }
 947
 948 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 949                                                   struct sk_buff *skb,
 950                                                   bool *skip_hw_vlan)
 951 {
 952         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 953         unsigned int eth_hdr_len;
 954         struct iphdr *ip;
 955
 956         /* For padded packets, BE HW modifies tot_len field in IP header
 957          * incorrecly when VLAN tag is inserted by HW.
 958          * For padded packets, Lancer computes incorrect checksum.
 959          */
 960         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 961                                                 VLAN_ETH_HLEN : ETH_HLEN;
 962         if (skb->len <= 60 &&
 963             (lancer_chip(adapter) || skb_vlan_tag_present(skb)) &&
 964             is_ipv4_pkt(skb)) {
 965                 ip = (struct iphdr *)ip_hdr(skb);
 966                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 967         }
 968
 969         /* If vlan tag is already inlined in the packet, skip HW VLAN
 970          * tagging in pvid-tagging mode
 971          */
 972         if (be_pvid_tagging_enabled(adapter) &&
 973             veh->h_vlan_proto == htons(ETH_P_8021Q))
 974                 *skip_hw_vlan = true;
 975
 976         /* HW has a bug wherein it will calculate CSUM for VLAN
 977          * pkts even though it is disabled.
 978          * Manually insert VLAN in pkt.
 979          */
 980         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 981             skb_vlan_tag_present(skb)) {
 982                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 983                 if (unlikely(!skb))
 984                         goto err;
 985         }
 986
 987         /* HW may lockup when VLAN HW tagging is requested on
 988          * certain ipv6 packets. Drop such pkts if the HW workaround to
 989          * skip HW tagging is not enabled by FW.
 990          */
 991         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 992                      (adapter->pvid || adapter->qnq_vid) &&
 993                      !qnq_async_evt_rcvd(adapter)))
 994                 goto tx_drop;
 995
 996         /* Manual VLAN tag insertion to prevent:
 997          * ASIC lockup when the ASIC inserts VLAN tag into
 998          * certain ipv6 packets. Insert VLAN tags in driver,
 999          * and set event, completion, vlan bits accordingly
1000          * in the Tx WRB.
1001          */
1002         if (be_ipv6_tx_stall_chk(adapter, skb) &&
1003             be_vlan_tag_tx_chk(adapter, skb)) {
1004                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
1005                 if (unlikely(!skb))
1006                         goto err;
1007         }
1008
1009         return skb;
1010 tx_drop:
1011         dev_kfree_skb_any(skb);
1012 err:
1013         return NULL;
1014 }
1015
1016 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1017                                            struct sk_buff *skb,
1018                                            bool *skip_hw_vlan)
1019 {
1020         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1021          * less may cause a transmit stall on that port. So the work-around is
1022          * to pad short packets (<= 32 bytes) to a 36-byte length.
1023          */
1024         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1025                 if (skb_put_padto(skb, 36))
1026                         return NULL;
1027         }
1028
1029         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1030                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1031                 if (!skb)
1032                         return NULL;
1033         }
1034
1035         return skb;
1036 }
1037
1038 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)
1039 {
1040         struct be_queue_info *txq = &txo->q;
1041         struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);
1042
1043         /* Mark the last request eventable if it hasn't been marked already */
1044         if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT)))
1045                 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL);
1046
1047         /* compose a dummy wrb if there are odd set of wrbs to notify */
1048         if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {
1049                 wrb_fill_dummy(queue_head_node(txq));
1050                 queue_head_inc(txq);
1051                 atomic_inc(&txq->used);
1052                 txo->pend_wrb_cnt++;
1053                 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK <<
1054                                            TX_HDR_WRB_NUM_SHIFT);
1055                 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<
1056                                           TX_HDR_WRB_NUM_SHIFT);
1057         }
1058         be_txq_notify(adapter, txo, txo->pend_wrb_cnt);
1059         txo->pend_wrb_cnt = 0;
1060 }
1061
1062 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1063 {
1064         bool skip_hw_vlan = false, flush = !skb->xmit_more;
1065         struct be_adapter *adapter = netdev_priv(netdev);
1066         u16 q_idx = skb_get_queue_mapping(skb);
1067         struct be_tx_obj *txo = &adapter->tx_obj[q_idx];
1068         struct be_queue_info *txq = &txo->q;
1069         u16 wrb_cnt;
1070
1071         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1072         if (unlikely(!skb))
1073                 goto drop;
1074
1075         wrb_cnt = be_xmit_enqueue(adapter, txo, skb, skip_hw_vlan);
1076         if (unlikely(!wrb_cnt)) {
1077                 dev_kfree_skb_any(skb);
1078                 goto drop;
1079         }
1080
1081         if ((atomic_read(&txq->used) + BE_MAX_TX_FRAG_COUNT) >= txq->len) {
1082                 netif_stop_subqueue(netdev, q_idx);
1083                 tx_stats(txo)->tx_stops++;
1084         }
1085
1086         if (flush || __netif_subqueue_stopped(netdev, q_idx))
1087                 be_xmit_flush(adapter, txo);
1088
1089         return NETDEV_TX_OK;
1090 drop:
1091         tx_stats(txo)->tx_drv_drops++;
1092         /* Flush the already enqueued tx requests */
1093         if (flush && txo->pend_wrb_cnt)
1094                 be_xmit_flush(adapter, txo);
1095
1096         return NETDEV_TX_OK;
1097 }
1098
1099 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1100 {
1101         struct be_adapter *adapter = netdev_priv(netdev);
1102         struct device *dev = &adapter->pdev->dev;
1103
1104         if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) {
1105                 dev_info(dev, "MTU must be between %d and %d bytes\n",
1106                          BE_MIN_MTU, BE_MAX_MTU);
1107                 return -EINVAL;
1108         }
1109
1110         dev_info(dev, "MTU changed from %d to %d bytes\n",
1111                  netdev->mtu, new_mtu);
1112         netdev->mtu = new_mtu;
1113         return 0;
1114 }
1115
1116 static inline bool be_in_all_promisc(struct be_adapter *adapter)
1117 {
1118         return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) ==
1119                         BE_IF_FLAGS_ALL_PROMISCUOUS;
1120 }
1121
1122 static int be_set_vlan_promisc(struct be_adapter *adapter)
1123 {
1124         struct device *dev = &adapter->pdev->dev;
1125         int status;
1126
1127         if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS)
1128                 return 0;
1129
1130         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON);
1131         if (!status) {
1132                 dev_info(dev, "Enabled VLAN promiscuous mode\n");
1133                 adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS;
1134         } else {
1135                 dev_err(dev, "Failed to enable VLAN promiscuous mode\n");
1136         }
1137         return status;
1138 }
1139
1140 static int be_clear_vlan_promisc(struct be_adapter *adapter)
1141 {
1142         struct device *dev = &adapter->pdev->dev;
1143         int status;
1144
1145         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF);
1146         if (!status) {
1147                 dev_info(dev, "Disabling VLAN promiscuous mode\n");
1148                 adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
1149         }
1150         return status;
1151 }
1152
1153 /*
1154  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1155  * If the user configures more, place BE in vlan promiscuous mode.
1156  */
1157 static int be_vid_config(struct be_adapter *adapter)
1158 {
1159         struct device *dev = &adapter->pdev->dev;
1160         u16 vids[BE_NUM_VLANS_SUPPORTED];
1161         u16 num = 0, i = 0;
1162         int status = 0;
1163
1164         /* No need to further configure vids if in promiscuous mode */
1165         if (be_in_all_promisc(adapter))
1166                 return 0;
1167
1168         if (adapter->vlans_added > be_max_vlans(adapter))
1169                 return be_set_vlan_promisc(adapter);
1170
1171         /* Construct VLAN Table to give to HW */
1172         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1173                 vids[num++] = cpu_to_le16(i);
1174
1175         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num);
1176         if (status) {
1177                 dev_err(dev, "Setting HW VLAN filtering failed\n");
1178                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1179                 if (addl_status(status) ==
1180                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1181                         return be_set_vlan_promisc(adapter);
1182         } else if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
1183                 status = be_clear_vlan_promisc(adapter);
1184         }
1185         return status;
1186 }
1187
1188 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1189 {
1190         struct be_adapter *adapter = netdev_priv(netdev);
1191         int status = 0;
1192
1193         /* Packets with VID 0 are always received by Lancer by default */
1194         if (lancer_chip(adapter) && vid == 0)
1195                 return status;
1196
1197         if (test_bit(vid, adapter->vids))
1198                 return status;
1199
1200         set_bit(vid, adapter->vids);
1201         adapter->vlans_added++;
1202
1203         status = be_vid_config(adapter);
1204         if (status) {
1205                 adapter->vlans_added--;
1206                 clear_bit(vid, adapter->vids);
1207         }
1208
1209         return status;
1210 }
1211
1212 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1213 {
1214         struct be_adapter *adapter = netdev_priv(netdev);
1215
1216         /* Packets with VID 0 are always received by Lancer by default */
1217         if (lancer_chip(adapter) && vid == 0)
1218                 return 0;
1219
1220         clear_bit(vid, adapter->vids);
1221         adapter->vlans_added--;
1222
1223         return be_vid_config(adapter);
1224 }
1225
1226 static void be_clear_all_promisc(struct be_adapter *adapter)
1227 {
1228         be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, OFF);
1229         adapter->if_flags &= ~BE_IF_FLAGS_ALL_PROMISCUOUS;
1230 }
1231
1232 static void be_set_all_promisc(struct be_adapter *adapter)
1233 {
1234         be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON);
1235         adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS;
1236 }
1237
1238 static void be_set_mc_promisc(struct be_adapter *adapter)
1239 {
1240         int status;
1241
1242         if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS)
1243                 return;
1244
1245         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON);
1246         if (!status)
1247                 adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS;
1248 }
1249
1250 static void be_set_mc_list(struct be_adapter *adapter)
1251 {
1252         int status;
1253
1254         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON);
1255         if (!status)
1256                 adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS;
1257         else
1258                 be_set_mc_promisc(adapter);
1259 }
1260
1261 static void be_set_uc_list(struct be_adapter *adapter)
1262 {
1263         struct netdev_hw_addr *ha;
1264         int i = 1; /* First slot is claimed by the Primary MAC */
1265
1266         for (; adapter->uc_macs > 0; adapter->uc_macs--, i++)
1267                 be_cmd_pmac_del(adapter, adapter->if_handle,
1268                                 adapter->pmac_id[i], 0);
1269
1270         if (netdev_uc_count(adapter->netdev) > be_max_uc(adapter)) {
1271                 be_set_all_promisc(adapter);
1272                 return;
1273         }
1274
1275         netdev_for_each_uc_addr(ha, adapter->netdev) {
1276                 adapter->uc_macs++; /* First slot is for Primary MAC */
1277                 be_cmd_pmac_add(adapter, (u8 *)ha->addr, adapter->if_handle,
1278                                 &adapter->pmac_id[adapter->uc_macs], 0);
1279         }
1280 }
1281
1282 static void be_clear_uc_list(struct be_adapter *adapter)
1283 {
1284         int i;
1285
1286         for (i = 1; i < (adapter->uc_macs + 1); i++)
1287                 be_cmd_pmac_del(adapter, adapter->if_handle,
1288                                 adapter->pmac_id[i], 0);
1289         adapter->uc_macs = 0;
1290 }
1291
1292 static void be_set_rx_mode(struct net_device *netdev)
1293 {
1294         struct be_adapter *adapter = netdev_priv(netdev);
1295
1296         if (netdev->flags & IFF_PROMISC) {
1297                 be_set_all_promisc(adapter);
1298                 return;
1299         }
1300
1301         /* Interface was previously in promiscuous mode; disable it */
1302         if (be_in_all_promisc(adapter)) {
1303                 be_clear_all_promisc(adapter);
1304                 if (adapter->vlans_added)
1305                         be_vid_config(adapter);
1306         }
1307
1308         /* Enable multicast promisc if num configured exceeds what we support */
1309         if (netdev->flags & IFF_ALLMULTI ||
1310             netdev_mc_count(netdev) > be_max_mc(adapter)) {
1311                 be_set_mc_promisc(adapter);
1312                 return;
1313         }
1314
1315         if (netdev_uc_count(netdev) != adapter->uc_macs)
1316                 be_set_uc_list(adapter);
1317
1318         be_set_mc_list(adapter);
1319 }
1320
1321 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1322 {
1323         struct be_adapter *adapter = netdev_priv(netdev);
1324         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1325         int status;
1326
1327         if (!sriov_enabled(adapter))
1328                 return -EPERM;
1329
1330         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1331                 return -EINVAL;
1332
1333         /* Proceed further only if user provided MAC is different
1334          * from active MAC
1335          */
1336         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1337                 return 0;
1338
1339         if (BEx_chip(adapter)) {
1340                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1341                                 vf + 1);
1342
1343                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1344                                          &vf_cfg->pmac_id, vf + 1);
1345         } else {
1346                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1347                                         vf + 1);
1348         }
1349
1350         if (status) {
1351                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1352                         mac, vf, status);
1353                 return be_cmd_status(status);
1354         }
1355
1356         ether_addr_copy(vf_cfg->mac_addr, mac);
1357
1358         return 0;
1359 }
1360
1361 static int be_get_vf_config(struct net_device *netdev, int vf,
1362                             struct ifla_vf_info *vi)
1363 {
1364         struct be_adapter *adapter = netdev_priv(netdev);
1365         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1366
1367         if (!sriov_enabled(adapter))
1368                 return -EPERM;
1369
1370         if (vf >= adapter->num_vfs)
1371                 return -EINVAL;
1372
1373         vi->vf = vf;
1374         vi->max_tx_rate = vf_cfg->tx_rate;
1375         vi->min_tx_rate = 0;
1376         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1377         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1378         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1379         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1380
1381         return 0;
1382 }
1383
1384 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1385 {
1386         struct be_adapter *adapter = netdev_priv(netdev);
1387         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1388         int status = 0;
1389
1390         if (!sriov_enabled(adapter))
1391                 return -EPERM;
1392
1393         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1394                 return -EINVAL;
1395
1396         if (vlan || qos) {
1397                 vlan |= qos << VLAN_PRIO_SHIFT;
1398                 if (vf_cfg->vlan_tag != vlan)
1399                         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1,
1400                                                        vf_cfg->if_handle, 0);
1401         } else {
1402                 /* Reset Transparent Vlan Tagging. */
1403                 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID,
1404                                                vf + 1, vf_cfg->if_handle, 0);
1405         }
1406
1407         if (status) {
1408                 dev_err(&adapter->pdev->dev,
1409                         "VLAN %d config on VF %d failed : %#x\n", vlan,
1410                         vf, status);
1411                 return be_cmd_status(status);
1412         }
1413
1414         vf_cfg->vlan_tag = vlan;
1415
1416         return 0;
1417 }
1418
1419 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1420                              int min_tx_rate, int max_tx_rate)
1421 {
1422         struct be_adapter *adapter = netdev_priv(netdev);
1423         struct device *dev = &adapter->pdev->dev;
1424         int percent_rate, status = 0;
1425         u16 link_speed = 0;
1426         u8 link_status;
1427
1428         if (!sriov_enabled(adapter))
1429                 return -EPERM;
1430
1431         if (vf >= adapter->num_vfs)
1432                 return -EINVAL;
1433
1434         if (min_tx_rate)
1435                 return -EINVAL;
1436
1437         if (!max_tx_rate)
1438                 goto config_qos;
1439
1440         status = be_cmd_link_status_query(adapter, &link_speed,
1441                                           &link_status, 0);
1442         if (status)
1443                 goto err;
1444
1445         if (!link_status) {
1446                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1447                 status = -ENETDOWN;
1448                 goto err;
1449         }
1450
1451         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1452                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1453                         link_speed);
1454                 status = -EINVAL;
1455                 goto err;
1456         }
1457
1458         /* On Skyhawk the QOS setting must be done only as a % value */
1459         percent_rate = link_speed / 100;
1460         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1461                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1462                         percent_rate);
1463                 status = -EINVAL;
1464                 goto err;
1465         }
1466
1467 config_qos:
1468         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1469         if (status)
1470                 goto err;
1471
1472         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1473         return 0;
1474
1475 err:
1476         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1477                 max_tx_rate, vf);
1478         return be_cmd_status(status);
1479 }
1480
1481 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1482                                 int link_state)
1483 {
1484         struct be_adapter *adapter = netdev_priv(netdev);
1485         int status;
1486
1487         if (!sriov_enabled(adapter))
1488                 return -EPERM;
1489
1490         if (vf >= adapter->num_vfs)
1491                 return -EINVAL;
1492
1493         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1494         if (status) {
1495                 dev_err(&adapter->pdev->dev,
1496                         "Link state change on VF %d failed: %#x\n", vf, status);
1497                 return be_cmd_status(status);
1498         }
1499
1500         adapter->vf_cfg[vf].plink_tracking = link_state;
1501
1502         return 0;
1503 }
1504
1505 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1506                           ulong now)
1507 {
1508         aic->rx_pkts_prev = rx_pkts;
1509         aic->tx_reqs_prev = tx_pkts;
1510         aic->jiffies = now;
1511 }
1512
1513 static void be_eqd_update(struct be_adapter *adapter)
1514 {
1515         struct be_set_eqd set_eqd[MAX_EVT_QS];
1516         int eqd, i, num = 0, start;
1517         struct be_aic_obj *aic;
1518         struct be_eq_obj *eqo;
1519         struct be_rx_obj *rxo;
1520         struct be_tx_obj *txo;
1521         u64 rx_pkts, tx_pkts;
1522         ulong now;
1523         u32 pps, delta;
1524
1525         for_all_evt_queues(adapter, eqo, i) {
1526                 aic = &adapter->aic_obj[eqo->idx];
1527                 if (!aic->enable) {
1528                         if (aic->jiffies)
1529                                 aic->jiffies = 0;
1530                         eqd = aic->et_eqd;
1531                         goto modify_eqd;
1532                 }
1533
1534                 rxo = &adapter->rx_obj[eqo->idx];
1535                 do {
1536                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1537                         rx_pkts = rxo->stats.rx_pkts;
1538                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1539
1540                 txo = &adapter->tx_obj[eqo->idx];
1541                 do {
1542                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1543                         tx_pkts = txo->stats.tx_reqs;
1544                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1545
1546                 /* Skip, if wrapped around or first calculation */
1547                 now = jiffies;
1548                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1549                     rx_pkts < aic->rx_pkts_prev ||
1550                     tx_pkts < aic->tx_reqs_prev) {
1551                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1552                         continue;
1553                 }
1554
1555                 delta = jiffies_to_msecs(now - aic->jiffies);
1556                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1557                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1558                 eqd = (pps / 15000) << 2;
1559
1560                 if (eqd < 8)
1561                         eqd = 0;
1562                 eqd = min_t(u32, eqd, aic->max_eqd);
1563                 eqd = max_t(u32, eqd, aic->min_eqd);
1564
1565                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1566 modify_eqd:
1567                 if (eqd != aic->prev_eqd) {
1568                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1569                         set_eqd[num].eq_id = eqo->q.id;
1570                         aic->prev_eqd = eqd;
1571                         num++;
1572                 }
1573         }
1574
1575         if (num)
1576                 be_cmd_modify_eqd(adapter, set_eqd, num);
1577 }
1578
1579 static void be_rx_stats_update(struct be_rx_obj *rxo,
1580                                struct be_rx_compl_info *rxcp)
1581 {
1582         struct be_rx_stats *stats = rx_stats(rxo);
1583
1584         u64_stats_update_begin(&stats->sync);
1585         stats->rx_compl++;
1586         stats->rx_bytes += rxcp->pkt_size;
1587         stats->rx_pkts++;
1588         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1589                 stats->rx_mcast_pkts++;
1590         if (rxcp->err)
1591                 stats->rx_compl_err++;
1592         u64_stats_update_end(&stats->sync);
1593 }
1594
1595 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1596 {
1597         /* L4 checksum is not reliable for non TCP/UDP packets.
1598          * Also ignore ipcksm for ipv6 pkts
1599          */
1600         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1601                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1602 }
1603
1604 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1605 {
1606         struct be_adapter *adapter = rxo->adapter;
1607         struct be_rx_page_info *rx_page_info;
1608         struct be_queue_info *rxq = &rxo->q;
1609         u16 frag_idx = rxq->tail;
1610
1611         rx_page_info = &rxo->page_info_tbl[frag_idx];
1612         BUG_ON(!rx_page_info->page);
1613
1614         if (rx_page_info->last_frag) {
1615                 dma_unmap_page(&adapter->pdev->dev,
1616                                dma_unmap_addr(rx_page_info, bus),
1617                                adapter->big_page_size, DMA_FROM_DEVICE);
1618                 rx_page_info->last_frag = false;
1619         } else {
1620                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1621                                         dma_unmap_addr(rx_page_info, bus),
1622                                         rx_frag_size, DMA_FROM_DEVICE);
1623         }
1624
1625         queue_tail_inc(rxq);
1626         atomic_dec(&rxq->used);
1627         return rx_page_info;
1628 }
1629
1630 /* Throwaway the data in the Rx completion */
1631 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1632                                 struct be_rx_compl_info *rxcp)
1633 {
1634         struct be_rx_page_info *page_info;
1635         u16 i, num_rcvd = rxcp->num_rcvd;
1636
1637         for (i = 0; i < num_rcvd; i++) {
1638                 page_info = get_rx_page_info(rxo);
1639                 put_page(page_info->page);
1640                 memset(page_info, 0, sizeof(*page_info));
1641         }
1642 }
1643
1644 /*
1645  * skb_fill_rx_data forms a complete skb for an ether frame
1646  * indicated by rxcp.
1647  */
1648 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1649                              struct be_rx_compl_info *rxcp)
1650 {
1651         struct be_rx_page_info *page_info;
1652         u16 i, j;
1653         u16 hdr_len, curr_frag_len, remaining;
1654         u8 *start;
1655
1656         page_info = get_rx_page_info(rxo);
1657         start = page_address(page_info->page) + page_info->page_offset;
1658         prefetch(start);
1659
1660         /* Copy data in the first descriptor of this completion */
1661         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1662
1663         skb->len = curr_frag_len;
1664         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1665                 memcpy(skb->data, start, curr_frag_len);
1666                 /* Complete packet has now been moved to data */
1667                 put_page(page_info->page);
1668                 skb->data_len = 0;
1669                 skb->tail += curr_frag_len;
1670         } else {
1671                 hdr_len = ETH_HLEN;
1672                 memcpy(skb->data, start, hdr_len);
1673                 skb_shinfo(skb)->nr_frags = 1;
1674                 skb_frag_set_page(skb, 0, page_info->page);
1675                 skb_shinfo(skb)->frags[0].page_offset =
1676                                         page_info->page_offset + hdr_len;
1677                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1678                                   curr_frag_len - hdr_len);
1679                 skb->data_len = curr_frag_len - hdr_len;
1680                 skb->truesize += rx_frag_size;
1681                 skb->tail += hdr_len;
1682         }
1683         page_info->page = NULL;
1684
1685         if (rxcp->pkt_size <= rx_frag_size) {
1686                 BUG_ON(rxcp->num_rcvd != 1);
1687                 return;
1688         }
1689
1690         /* More frags present for this completion */
1691         remaining = rxcp->pkt_size - curr_frag_len;
1692         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1693                 page_info = get_rx_page_info(rxo);
1694                 curr_frag_len = min(remaining, rx_frag_size);
1695
1696                 /* Coalesce all frags from the same physical page in one slot */
1697                 if (page_info->page_offset == 0) {
1698                         /* Fresh page */
1699                         j++;
1700                         skb_frag_set_page(skb, j, page_info->page);
1701                         skb_shinfo(skb)->frags[j].page_offset =
1702                                                         page_info->page_offset;
1703                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1704                         skb_shinfo(skb)->nr_frags++;
1705                 } else {
1706                         put_page(page_info->page);
1707                 }
1708
1709                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1710                 skb->len += curr_frag_len;
1711                 skb->data_len += curr_frag_len;
1712                 skb->truesize += rx_frag_size;
1713                 remaining -= curr_frag_len;
1714                 page_info->page = NULL;
1715         }
1716         BUG_ON(j > MAX_SKB_FRAGS);
1717 }
1718
1719 /* Process the RX completion indicated by rxcp when GRO is disabled */
1720 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1721                                 struct be_rx_compl_info *rxcp)
1722 {
1723         struct be_adapter *adapter = rxo->adapter;
1724         struct net_device *netdev = adapter->netdev;
1725         struct sk_buff *skb;
1726
1727         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1728         if (unlikely(!skb)) {
1729                 rx_stats(rxo)->rx_drops_no_skbs++;
1730                 be_rx_compl_discard(rxo, rxcp);
1731                 return;
1732         }
1733
1734         skb_fill_rx_data(rxo, skb, rxcp);
1735
1736         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1737                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1738         else
1739                 skb_checksum_none_assert(skb);
1740
1741         skb->protocol = eth_type_trans(skb, netdev);
1742         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1743         if (netdev->features & NETIF_F_RXHASH)
1744                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1745
1746         skb->csum_level = rxcp->tunneled;
1747         skb_mark_napi_id(skb, napi);
1748
1749         if (rxcp->vlanf)
1750                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1751
1752         netif_receive_skb(skb);
1753 }
1754
1755 /* Process the RX completion indicated by rxcp when GRO is enabled */
1756 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1757                                     struct napi_struct *napi,
1758                                     struct be_rx_compl_info *rxcp)
1759 {
1760         struct be_adapter *adapter = rxo->adapter;
1761         struct be_rx_page_info *page_info;
1762         struct sk_buff *skb = NULL;
1763         u16 remaining, curr_frag_len;
1764         u16 i, j;
1765
1766         skb = napi_get_frags(napi);
1767         if (!skb) {
1768                 be_rx_compl_discard(rxo, rxcp);
1769                 return;
1770         }
1771
1772         remaining = rxcp->pkt_size;
1773         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1774                 page_info = get_rx_page_info(rxo);
1775
1776                 curr_frag_len = min(remaining, rx_frag_size);
1777
1778                 /* Coalesce all frags from the same physical page in one slot */
1779                 if (i == 0 || page_info->page_offset == 0) {
1780                         /* First frag or Fresh page */
1781                         j++;
1782                         skb_frag_set_page(skb, j, page_info->page);
1783                         skb_shinfo(skb)->frags[j].page_offset =
1784                                                         page_info->page_offset;
1785                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1786                 } else {
1787                         put_page(page_info->page);
1788                 }
1789                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1790                 skb->truesize += rx_frag_size;
1791                 remaining -= curr_frag_len;
1792                 memset(page_info, 0, sizeof(*page_info));
1793         }
1794         BUG_ON(j > MAX_SKB_FRAGS);
1795
1796         skb_shinfo(skb)->nr_frags = j + 1;
1797         skb->len = rxcp->pkt_size;
1798         skb->data_len = rxcp->pkt_size;
1799         skb->ip_summed = CHECKSUM_UNNECESSARY;
1800         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1801         if (adapter->netdev->features & NETIF_F_RXHASH)
1802                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1803
1804         skb->csum_level = rxcp->tunneled;
1805         skb_mark_napi_id(skb, napi);
1806
1807         if (rxcp->vlanf)
1808                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1809
1810         napi_gro_frags(napi);
1811 }
1812
1813 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1814                                  struct be_rx_compl_info *rxcp)
1815 {
1816         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
1817         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
1818         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
1819         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
1820         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
1821         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
1822         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
1823         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
1824         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
1825         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
1826         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
1827         if (rxcp->vlanf) {
1828                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
1829                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
1830         }
1831         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
1832         rxcp->tunneled =
1833                 GET_RX_COMPL_V1_BITS(tunneled, compl);
1834 }
1835
1836 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1837                                  struct be_rx_compl_info *rxcp)
1838 {
1839         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
1840         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
1841         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
1842         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
1843         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
1844         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
1845         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
1846         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
1847         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
1848         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
1849         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
1850         if (rxcp->vlanf) {
1851                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
1852                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
1853         }
1854         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
1855         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
1856 }
1857
1858 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1859 {
1860         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1861         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1862         struct be_adapter *adapter = rxo->adapter;
1863
1864         /* For checking the valid bit it is Ok to use either definition as the
1865          * valid bit is at the same position in both v0 and v1 Rx compl */
1866         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1867                 return NULL;
1868
1869         rmb();
1870         be_dws_le_to_cpu(compl, sizeof(*compl));
1871
1872         if (adapter->be3_native)
1873                 be_parse_rx_compl_v1(compl, rxcp);
1874         else
1875                 be_parse_rx_compl_v0(compl, rxcp);
1876
1877         if (rxcp->ip_frag)
1878                 rxcp->l4_csum = 0;
1879
1880         if (rxcp->vlanf) {
1881                 /* In QNQ modes, if qnq bit is not set, then the packet was
1882                  * tagged only with the transparent outer vlan-tag and must
1883                  * not be treated as a vlan packet by host
1884                  */
1885                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1886                         rxcp->vlanf = 0;
1887
1888                 if (!lancer_chip(adapter))
1889                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1890
1891                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1892                     !test_bit(rxcp->vlan_tag, adapter->vids))
1893                         rxcp->vlanf = 0;
1894         }
1895
1896         /* As the compl has been parsed, reset it; we wont touch it again */
1897         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1898
1899         queue_tail_inc(&rxo->cq);
1900         return rxcp;
1901 }
1902
1903 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1904 {
1905         u32 order = get_order(size);
1906
1907         if (order > 0)
1908                 gfp |= __GFP_COMP;
1909         return  alloc_pages(gfp, order);
1910 }
1911
1912 /*
1913  * Allocate a page, split it to fragments of size rx_frag_size and post as
1914  * receive buffers to BE
1915  */
1916 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
1917 {
1918         struct be_adapter *adapter = rxo->adapter;
1919         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1920         struct be_queue_info *rxq = &rxo->q;
1921         struct page *pagep = NULL;
1922         struct device *dev = &adapter->pdev->dev;
1923         struct be_eth_rx_d *rxd;
1924         u64 page_dmaaddr = 0, frag_dmaaddr;
1925         u32 posted, page_offset = 0, notify = 0;
1926
1927         page_info = &rxo->page_info_tbl[rxq->head];
1928         for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
1929                 if (!pagep) {
1930                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1931                         if (unlikely(!pagep)) {
1932                                 rx_stats(rxo)->rx_post_fail++;
1933                                 break;
1934                         }
1935                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1936                                                     adapter->big_page_size,
1937                                                     DMA_FROM_DEVICE);
1938                         if (dma_mapping_error(dev, page_dmaaddr)) {
1939                                 put_page(pagep);
1940                                 pagep = NULL;
1941                                 adapter->drv_stats.dma_map_errors++;
1942                                 break;
1943                         }
1944                         page_offset = 0;
1945                 } else {
1946                         get_page(pagep);
1947                         page_offset += rx_frag_size;
1948                 }
1949                 page_info->page_offset = page_offset;
1950                 page_info->page = pagep;
1951
1952                 rxd = queue_head_node(rxq);
1953                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
1954                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
1955                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
1956
1957                 /* Any space left in the current big page for another frag? */
1958                 if ((page_offset + rx_frag_size + rx_frag_size) >
1959                                         adapter->big_page_size) {
1960                         pagep = NULL;
1961                         page_info->last_frag = true;
1962                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
1963                 } else {
1964                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
1965                 }
1966
1967                 prev_page_info = page_info;
1968                 queue_head_inc(rxq);
1969                 page_info = &rxo->page_info_tbl[rxq->head];
1970         }
1971
1972         /* Mark the last frag of a page when we break out of the above loop
1973          * with no more slots available in the RXQ
1974          */
1975         if (pagep) {
1976                 prev_page_info->last_frag = true;
1977                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
1978         }
1979
1980         if (posted) {
1981                 atomic_add(posted, &rxq->used);
1982                 if (rxo->rx_post_starved)
1983                         rxo->rx_post_starved = false;
1984                 do {
1985                         notify = min(256u, posted);
1986                         be_rxq_notify(adapter, rxq->id, notify);
1987                         posted -= notify;
1988                 } while (posted);
1989         } else if (atomic_read(&rxq->used) == 0) {
1990                 /* Let be_worker replenish when memory is available */
1991                 rxo->rx_post_starved = true;
1992         }
1993 }
1994
1995 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
1996 {
1997         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
1998
1999         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
2000                 return NULL;
2001
2002         rmb();
2003         be_dws_le_to_cpu(txcp, sizeof(*txcp));
2004
2005         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
2006
2007         queue_tail_inc(tx_cq);
2008         return txcp;
2009 }
2010
2011 static u16 be_tx_compl_process(struct be_adapter *adapter,
2012                                struct be_tx_obj *txo, u16 last_index)
2013 {
2014         struct sk_buff **sent_skbs = txo->sent_skb_list;
2015         struct be_queue_info *txq = &txo->q;
2016         u16 frag_index, num_wrbs = 0;
2017         struct sk_buff *skb = NULL;
2018         bool unmap_skb_hdr = false;
2019         struct be_eth_wrb *wrb;
2020
2021         do {
2022                 if (sent_skbs[txq->tail]) {
2023                         /* Free skb from prev req */
2024                         if (skb)
2025                                 dev_consume_skb_any(skb);
2026                         skb = sent_skbs[txq->tail];
2027                         sent_skbs[txq->tail] = NULL;
2028                         queue_tail_inc(txq);  /* skip hdr wrb */
2029                         num_wrbs++;
2030                         unmap_skb_hdr = true;
2031                 }
2032                 wrb = queue_tail_node(txq);
2033                 frag_index = txq->tail;
2034                 unmap_tx_frag(&adapter->pdev->dev, wrb,
2035                               (unmap_skb_hdr && skb_headlen(skb)));
2036                 unmap_skb_hdr = false;
2037                 queue_tail_inc(txq);
2038                 num_wrbs++;
2039         } while (frag_index != last_index);
2040         dev_consume_skb_any(skb);
2041
2042         return num_wrbs;
2043 }
2044
2045 /* Return the number of events in the event queue */
2046 static inline int events_get(struct be_eq_obj *eqo)
2047 {
2048         struct be_eq_entry *eqe;
2049         int num = 0;
2050
2051         do {
2052                 eqe = queue_tail_node(&eqo->q);
2053                 if (eqe->evt == 0)
2054                         break;
2055
2056                 rmb();
2057                 eqe->evt = 0;
2058                 num++;
2059                 queue_tail_inc(&eqo->q);
2060         } while (true);
2061
2062         return num;
2063 }
2064
2065 /* Leaves the EQ is disarmed state */
2066 static void be_eq_clean(struct be_eq_obj *eqo)
2067 {
2068         int num = events_get(eqo);
2069
2070         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2071 }
2072
2073 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2074 {
2075         struct be_rx_page_info *page_info;
2076         struct be_queue_info *rxq = &rxo->q;
2077         struct be_queue_info *rx_cq = &rxo->cq;
2078         struct be_rx_compl_info *rxcp;
2079         struct be_adapter *adapter = rxo->adapter;
2080         int flush_wait = 0;
2081
2082         /* Consume pending rx completions.
2083          * Wait for the flush completion (identified by zero num_rcvd)
2084          * to arrive. Notify CQ even when there are no more CQ entries
2085          * for HW to flush partially coalesced CQ entries.
2086          * In Lancer, there is no need to wait for flush compl.
2087          */
2088         for (;;) {
2089                 rxcp = be_rx_compl_get(rxo);
2090                 if (!rxcp) {
2091                         if (lancer_chip(adapter))
2092                                 break;
2093
2094                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2095                                 dev_warn(&adapter->pdev->dev,
2096                                          "did not receive flush compl\n");
2097                                 break;
2098                         }
2099                         be_cq_notify(adapter, rx_cq->id, true, 0);
2100                         mdelay(1);
2101                 } else {
2102                         be_rx_compl_discard(rxo, rxcp);
2103                         be_cq_notify(adapter, rx_cq->id, false, 1);
2104                         if (rxcp->num_rcvd == 0)
2105                                 break;
2106                 }
2107         }
2108
2109         /* After cleanup, leave the CQ in unarmed state */
2110         be_cq_notify(adapter, rx_cq->id, false, 0);
2111
2112         /* Then free posted rx buffers that were not used */
2113         while (atomic_read(&rxq->used) > 0) {
2114                 page_info = get_rx_page_info(rxo);
2115                 put_page(page_info->page);
2116                 memset(page_info, 0, sizeof(*page_info));
2117         }
2118         BUG_ON(atomic_read(&rxq->used));
2119         rxq->tail = 0;
2120         rxq->head = 0;
2121 }
2122
2123 static void be_tx_compl_clean(struct be_adapter *adapter)
2124 {
2125         u16 end_idx, notified_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2126         struct device *dev = &adapter->pdev->dev;
2127         struct be_tx_obj *txo;
2128         struct be_queue_info *txq;
2129         struct be_eth_tx_compl *txcp;
2130         int i, pending_txqs;
2131
2132         /* Stop polling for compls when HW has been silent for 10ms */
2133         do {
2134                 pending_txqs = adapter->num_tx_qs;
2135
2136                 for_all_tx_queues(adapter, txo, i) {
2137                         cmpl = 0;
2138                         num_wrbs = 0;
2139                         txq = &txo->q;
2140                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2141                                 end_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2142                                 num_wrbs += be_tx_compl_process(adapter, txo,
2143                                                                 end_idx);
2144                                 cmpl++;
2145                         }
2146                         if (cmpl) {
2147                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2148                                 atomic_sub(num_wrbs, &txq->used);
2149                                 timeo = 0;
2150                         }
2151                         if (atomic_read(&txq->used) == txo->pend_wrb_cnt)
2152                                 pending_txqs--;
2153                 }
2154
2155                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2156                         break;
2157
2158                 mdelay(1);
2159         } while (true);
2160
2161         /* Free enqueued TX that was never notified to HW */
2162         for_all_tx_queues(adapter, txo, i) {
2163                 txq = &txo->q;
2164
2165                 if (atomic_read(&txq->used)) {
2166                         dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n",
2167                                  i, atomic_read(&txq->used));
2168                         notified_idx = txq->tail;
2169                         end_idx = txq->tail;
2170                         index_adv(&end_idx, atomic_read(&txq->used) - 1,
2171                                   txq->len);
2172                         /* Use the tx-compl process logic to handle requests
2173                          * that were not sent to the HW.
2174                          */
2175                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2176                         atomic_sub(num_wrbs, &txq->used);
2177                         BUG_ON(atomic_read(&txq->used));
2178                         txo->pend_wrb_cnt = 0;
2179                         /* Since hw was never notified of these requests,
2180                          * reset TXQ indices
2181                          */
2182                         txq->head = notified_idx;
2183                         txq->tail = notified_idx;
2184                 }
2185         }
2186 }
2187
2188 static void be_evt_queues_destroy(struct be_adapter *adapter)
2189 {
2190         struct be_eq_obj *eqo;
2191         int i;
2192
2193         for_all_evt_queues(adapter, eqo, i) {
2194                 if (eqo->q.created) {
2195                         be_eq_clean(eqo);
2196                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2197                         napi_hash_del(&eqo->napi);
2198                         netif_napi_del(&eqo->napi);
2199                 }
2200                 be_queue_free(adapter, &eqo->q);
2201         }
2202 }
2203
2204 static int be_evt_queues_create(struct be_adapter *adapter)
2205 {
2206         struct be_queue_info *eq;
2207         struct be_eq_obj *eqo;
2208         struct be_aic_obj *aic;
2209         int i, rc;
2210
2211         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2212                                     adapter->cfg_num_qs);
2213
2214         for_all_evt_queues(adapter, eqo, i) {
2215                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2216                                BE_NAPI_WEIGHT);
2217                 napi_hash_add(&eqo->napi);
2218                 aic = &adapter->aic_obj[i];
2219                 eqo->adapter = adapter;
2220                 eqo->idx = i;
2221                 aic->max_eqd = BE_MAX_EQD;
2222                 aic->enable = true;
2223
2224                 eq = &eqo->q;
2225                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2226                                     sizeof(struct be_eq_entry));
2227                 if (rc)
2228                         return rc;
2229
2230                 rc = be_cmd_eq_create(adapter, eqo);
2231                 if (rc)
2232                         return rc;
2233         }
2234         return 0;
2235 }
2236
2237 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2238 {
2239         struct be_queue_info *q;
2240
2241         q = &adapter->mcc_obj.q;
2242         if (q->created)
2243                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2244         be_queue_free(adapter, q);
2245
2246         q = &adapter->mcc_obj.cq;
2247         if (q->created)
2248                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2249         be_queue_free(adapter, q);
2250 }
2251
2252 /* Must be called only after TX qs are created as MCC shares TX EQ */
2253 static int be_mcc_queues_create(struct be_adapter *adapter)
2254 {
2255         struct be_queue_info *q, *cq;
2256
2257         cq = &adapter->mcc_obj.cq;
2258         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2259                            sizeof(struct be_mcc_compl)))
2260                 goto err;
2261
2262         /* Use the default EQ for MCC completions */
2263         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2264                 goto mcc_cq_free;
2265
2266         q = &adapter->mcc_obj.q;
2267         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2268                 goto mcc_cq_destroy;
2269
2270         if (be_cmd_mccq_create(adapter, q, cq))
2271                 goto mcc_q_free;
2272
2273         return 0;
2274
2275 mcc_q_free:
2276         be_queue_free(adapter, q);
2277 mcc_cq_destroy:
2278         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2279 mcc_cq_free:
2280         be_queue_free(adapter, cq);
2281 err:
2282         return -1;
2283 }
2284
2285 static void be_tx_queues_destroy(struct be_adapter *adapter)
2286 {
2287         struct be_queue_info *q;
2288         struct be_tx_obj *txo;
2289         u8 i;
2290
2291         for_all_tx_queues(adapter, txo, i) {
2292                 q = &txo->q;
2293                 if (q->created)
2294                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2295                 be_queue_free(adapter, q);
2296
2297                 q = &txo->cq;
2298                 if (q->created)
2299                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2300                 be_queue_free(adapter, q);
2301         }
2302 }
2303
2304 static int be_tx_qs_create(struct be_adapter *adapter)
2305 {
2306         struct be_queue_info *cq, *eq;
2307         struct be_tx_obj *txo;
2308         int status, i;
2309
2310         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2311
2312         for_all_tx_queues(adapter, txo, i) {
2313                 cq = &txo->cq;
2314                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2315                                         sizeof(struct be_eth_tx_compl));
2316                 if (status)
2317                         return status;
2318
2319                 u64_stats_init(&txo->stats.sync);
2320                 u64_stats_init(&txo->stats.sync_compl);
2321
2322                 /* If num_evt_qs is less than num_tx_qs, then more than
2323                  * one txq share an eq
2324                  */
2325                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2326                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2327                 if (status)
2328                         return status;
2329
2330                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2331                                         sizeof(struct be_eth_wrb));
2332                 if (status)
2333                         return status;
2334
2335                 status = be_cmd_txq_create(adapter, txo);
2336                 if (status)
2337                         return status;
2338         }
2339
2340         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2341                  adapter->num_tx_qs);
2342         return 0;
2343 }
2344
2345 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2346 {
2347         struct be_queue_info *q;
2348         struct be_rx_obj *rxo;
2349         int i;
2350
2351         for_all_rx_queues(adapter, rxo, i) {
2352                 q = &rxo->cq;
2353                 if (q->created)
2354                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2355                 be_queue_free(adapter, q);
2356         }
2357 }
2358
2359 static int be_rx_cqs_create(struct be_adapter *adapter)
2360 {
2361         struct be_queue_info *eq, *cq;
2362         struct be_rx_obj *rxo;
2363         int rc, i;
2364
2365         /* We can create as many RSS rings as there are EQs. */
2366         adapter->num_rx_qs = adapter->num_evt_qs;
2367
2368         /* We'll use RSS only if atleast 2 RSS rings are supported.
2369          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2370          */
2371         if (adapter->num_rx_qs > 1)
2372                 adapter->num_rx_qs++;
2373
2374         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2375         for_all_rx_queues(adapter, rxo, i) {
2376                 rxo->adapter = adapter;
2377                 cq = &rxo->cq;
2378                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2379                                     sizeof(struct be_eth_rx_compl));
2380                 if (rc)
2381                         return rc;
2382
2383                 u64_stats_init(&rxo->stats.sync);
2384                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2385                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2386                 if (rc)
2387                         return rc;
2388         }
2389
2390         dev_info(&adapter->pdev->dev,
2391                  "created %d RSS queue(s) and 1 default RX queue\n",
2392                  adapter->num_rx_qs - 1);
2393         return 0;
2394 }
2395
2396 static irqreturn_t be_intx(int irq, void *dev)
2397 {
2398         struct be_eq_obj *eqo = dev;
2399         struct be_adapter *adapter = eqo->adapter;
2400         int num_evts = 0;
2401
2402         /* IRQ is not expected when NAPI is scheduled as the EQ
2403          * will not be armed.
2404          * But, this can happen on Lancer INTx where it takes
2405          * a while to de-assert INTx or in BE2 where occasionaly
2406          * an interrupt may be raised even when EQ is unarmed.
2407          * If NAPI is already scheduled, then counting & notifying
2408          * events will orphan them.
2409          */
2410         if (napi_schedule_prep(&eqo->napi)) {
2411                 num_evts = events_get(eqo);
2412                 __napi_schedule(&eqo->napi);
2413                 if (num_evts)
2414                         eqo->spurious_intr = 0;
2415         }
2416         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2417
2418         /* Return IRQ_HANDLED only for the the first spurious intr
2419          * after a valid intr to stop the kernel from branding
2420          * this irq as a bad one!
2421          */
2422         if (num_evts || eqo->spurious_intr++ == 0)
2423                 return IRQ_HANDLED;
2424         else
2425                 return IRQ_NONE;
2426 }
2427
2428 static irqreturn_t be_msix(int irq, void *dev)
2429 {
2430         struct be_eq_obj *eqo = dev;
2431
2432         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2433         napi_schedule(&eqo->napi);
2434         return IRQ_HANDLED;
2435 }
2436
2437 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2438 {
2439         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2440 }
2441
2442 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2443                          int budget, int polling)
2444 {
2445         struct be_adapter *adapter = rxo->adapter;
2446         struct be_queue_info *rx_cq = &rxo->cq;
2447         struct be_rx_compl_info *rxcp;
2448         u32 work_done;
2449         u32 frags_consumed = 0;
2450
2451         for (work_done = 0; work_done < budget; work_done++) {
2452                 rxcp = be_rx_compl_get(rxo);
2453                 if (!rxcp)
2454                         break;
2455
2456                 /* Is it a flush compl that has no data */
2457                 if (unlikely(rxcp->num_rcvd == 0))
2458                         goto loop_continue;
2459
2460                 /* Discard compl with partial DMA Lancer B0 */
2461                 if (unlikely(!rxcp->pkt_size)) {
2462                         be_rx_compl_discard(rxo, rxcp);
2463                         goto loop_continue;
2464                 }
2465
2466                 /* On BE drop pkts that arrive due to imperfect filtering in
2467                  * promiscuous mode on some skews
2468                  */
2469                 if (unlikely(rxcp->port != adapter->port_num &&
2470                              !lancer_chip(adapter))) {
2471                         be_rx_compl_discard(rxo, rxcp);
2472                         goto loop_continue;
2473                 }
2474
2475                 /* Don't do gro when we're busy_polling */
2476                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2477                         be_rx_compl_process_gro(rxo, napi, rxcp);
2478                 else
2479                         be_rx_compl_process(rxo, napi, rxcp);
2480
2481 loop_continue:
2482                 frags_consumed += rxcp->num_rcvd;
2483                 be_rx_stats_update(rxo, rxcp);
2484         }
2485
2486         if (work_done) {
2487                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2488
2489                 /* When an rx-obj gets into post_starved state, just
2490                  * let be_worker do the posting.
2491                  */
2492                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2493                     !rxo->rx_post_starved)
2494                         be_post_rx_frags(rxo, GFP_ATOMIC,
2495                                          max_t(u32, MAX_RX_POST,
2496                                                frags_consumed));
2497         }
2498
2499         return work_done;
2500 }
2501
2502 static inline void be_update_tx_err(struct be_tx_obj *txo, u32 status)
2503 {
2504         switch (status) {
2505         case BE_TX_COMP_HDR_PARSE_ERR:
2506                 tx_stats(txo)->tx_hdr_parse_err++;
2507                 break;
2508         case BE_TX_COMP_NDMA_ERR:
2509                 tx_stats(txo)->tx_dma_err++;
2510                 break;
2511         case BE_TX_COMP_ACL_ERR:
2512                 tx_stats(txo)->tx_spoof_check_err++;
2513                 break;
2514         }
2515 }
2516
2517 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u32 status)
2518 {
2519         switch (status) {
2520         case LANCER_TX_COMP_LSO_ERR:
2521                 tx_stats(txo)->tx_tso_err++;
2522                 break;
2523         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2524         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2525                 tx_stats(txo)->tx_spoof_check_err++;
2526                 break;
2527         case LANCER_TX_COMP_QINQ_ERR:
2528                 tx_stats(txo)->tx_qinq_err++;
2529                 break;
2530         case LANCER_TX_COMP_PARITY_ERR:
2531                 tx_stats(txo)->tx_internal_parity_err++;
2532                 break;
2533         case LANCER_TX_COMP_DMA_ERR:
2534                 tx_stats(txo)->tx_dma_err++;
2535                 break;
2536         }
2537 }
2538
2539 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2540                           int idx)
2541 {
2542         struct be_eth_tx_compl *txcp;
2543         int num_wrbs = 0, work_done = 0;
2544         u32 compl_status;
2545         u16 last_idx;
2546
2547         while ((txcp = be_tx_compl_get(&txo->cq))) {
2548                 last_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2549                 num_wrbs += be_tx_compl_process(adapter, txo, last_idx);
2550                 work_done++;
2551
2552                 compl_status = GET_TX_COMPL_BITS(status, txcp);
2553                 if (compl_status) {
2554                         if (lancer_chip(adapter))
2555                                 lancer_update_tx_err(txo, compl_status);
2556                         else
2557                                 be_update_tx_err(txo, compl_status);
2558                 }
2559         }
2560
2561         if (work_done) {
2562                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2563                 atomic_sub(num_wrbs, &txo->q.used);
2564
2565                 /* As Tx wrbs have been freed up, wake up netdev queue
2566                  * if it was stopped due to lack of tx wrbs.  */
2567                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2568                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2569                         netif_wake_subqueue(adapter->netdev, idx);
2570                 }
2571
2572                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2573                 tx_stats(txo)->tx_compl += work_done;
2574                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2575         }
2576 }
2577
2578 #ifdef CONFIG_NET_RX_BUSY_POLL
2579 static inline bool be_lock_napi(struct be_eq_obj *eqo)
2580 {
2581         bool status = true;
2582
2583         spin_lock(&eqo->lock); /* BH is already disabled */
2584         if (eqo->state & BE_EQ_LOCKED) {
2585                 WARN_ON(eqo->state & BE_EQ_NAPI);
2586                 eqo->state |= BE_EQ_NAPI_YIELD;
2587                 status = false;
2588         } else {
2589                 eqo->state = BE_EQ_NAPI;
2590         }
2591         spin_unlock(&eqo->lock);
2592         return status;
2593 }
2594
2595 static inline void be_unlock_napi(struct be_eq_obj *eqo)
2596 {
2597         spin_lock(&eqo->lock); /* BH is already disabled */
2598
2599         WARN_ON(eqo->state & (BE_EQ_POLL | BE_EQ_NAPI_YIELD));
2600         eqo->state = BE_EQ_IDLE;
2601
2602         spin_unlock(&eqo->lock);
2603 }
2604
2605 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
2606 {
2607         bool status = true;
2608
2609         spin_lock_bh(&eqo->lock);
2610         if (eqo->state & BE_EQ_LOCKED) {
2611                 eqo->state |= BE_EQ_POLL_YIELD;
2612                 status = false;
2613         } else {
2614                 eqo->state |= BE_EQ_POLL;
2615         }
2616         spin_unlock_bh(&eqo->lock);
2617         return status;
2618 }
2619
2620 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
2621 {
2622         spin_lock_bh(&eqo->lock);
2623
2624         WARN_ON(eqo->state & (BE_EQ_NAPI));
2625         eqo->state = BE_EQ_IDLE;
2626
2627         spin_unlock_bh(&eqo->lock);
2628 }
2629
2630 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
2631 {
2632         spin_lock_init(&eqo->lock);
2633         eqo->state = BE_EQ_IDLE;
2634 }
2635
2636 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
2637 {
2638         local_bh_disable();
2639
2640         /* It's enough to just acquire napi lock on the eqo to stop
2641          * be_busy_poll() from processing any queueus.
2642          */
2643         while (!be_lock_napi(eqo))
2644                 mdelay(1);
2645
2646         local_bh_enable();
2647 }
2648
2649 #else /* CONFIG_NET_RX_BUSY_POLL */
2650
2651 static inline bool be_lock_napi(struct be_eq_obj *eqo)
2652 {
2653         return true;
2654 }
2655
2656 static inline void be_unlock_napi(struct be_eq_obj *eqo)
2657 {
2658 }
2659
2660 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
2661 {
2662         return false;
2663 }
2664
2665 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
2666 {
2667 }
2668
2669 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
2670 {
2671 }
2672
2673 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
2674 {
2675 }
2676 #endif /* CONFIG_NET_RX_BUSY_POLL */
2677
2678 int be_poll(struct napi_struct *napi, int budget)
2679 {
2680         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2681         struct be_adapter *adapter = eqo->adapter;
2682         int max_work = 0, work, i, num_evts;
2683         struct be_rx_obj *rxo;
2684         struct be_tx_obj *txo;
2685
2686         num_evts = events_get(eqo);
2687
2688         for_all_tx_queues_on_eq(adapter, eqo, txo, i)
2689                 be_process_tx(adapter, txo, i);
2690
2691         if (be_lock_napi(eqo)) {
2692                 /* This loop will iterate twice for EQ0 in which
2693                  * completions of the last RXQ (default one) are also processed
2694                  * For other EQs the loop iterates only once
2695                  */
2696                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2697                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2698                         max_work = max(work, max_work);
2699                 }
2700                 be_unlock_napi(eqo);
2701         } else {
2702                 max_work = budget;
2703         }
2704
2705         if (is_mcc_eqo(eqo))
2706                 be_process_mcc(adapter);
2707
2708         if (max_work < budget) {
2709                 napi_complete(napi);
2710                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2711         } else {
2712                 /* As we'll continue in polling mode, count and clear events */
2713                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2714         }
2715         return max_work;
2716 }
2717
2718 #ifdef CONFIG_NET_RX_BUSY_POLL
2719 static int be_busy_poll(struct napi_struct *napi)
2720 {
2721         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2722         struct be_adapter *adapter = eqo->adapter;
2723         struct be_rx_obj *rxo;
2724         int i, work = 0;
2725
2726         if (!be_lock_busy_poll(eqo))
2727                 return LL_FLUSH_BUSY;
2728
2729         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2730                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2731                 if (work)
2732                         break;
2733         }
2734
2735         be_unlock_busy_poll(eqo);
2736         return work;
2737 }
2738 #endif
2739
2740 void be_detect_error(struct be_adapter *adapter)
2741 {
2742         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2743         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2744         u32 i;
2745         bool error_detected = false;
2746         struct device *dev = &adapter->pdev->dev;
2747         struct net_device *netdev = adapter->netdev;
2748
2749         if (be_hw_error(adapter))
2750                 return;
2751
2752         if (lancer_chip(adapter)) {
2753                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2754                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2755                         sliport_err1 = ioread32(adapter->db +
2756                                                 SLIPORT_ERROR1_OFFSET);
2757                         sliport_err2 = ioread32(adapter->db +
2758                                                 SLIPORT_ERROR2_OFFSET);
2759                         adapter->hw_error = true;
2760                         /* Do not log error messages if its a FW reset */
2761                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2762                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2763                                 dev_info(dev, "Firmware update in progress\n");
2764                         } else {
2765                                 error_detected = true;
2766                                 dev_err(dev, "Error detected in the card\n");
2767                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2768                                         sliport_status);
2769                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2770                                         sliport_err1);
2771                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2772                                         sliport_err2);
2773                         }
2774                 }
2775         } else {
2776                 pci_read_config_dword(adapter->pdev,
2777                                       PCICFG_UE_STATUS_LOW, &ue_lo);
2778                 pci_read_config_dword(adapter->pdev,
2779                                       PCICFG_UE_STATUS_HIGH, &ue_hi);
2780                 pci_read_config_dword(adapter->pdev,
2781                                       PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2782                 pci_read_config_dword(adapter->pdev,
2783                                       PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2784
2785                 ue_lo = (ue_lo & ~ue_lo_mask);
2786                 ue_hi = (ue_hi & ~ue_hi_mask);
2787
2788                 /* On certain platforms BE hardware can indicate spurious UEs.
2789                  * Allow HW to stop working completely in case of a real UE.
2790                  * Hence not setting the hw_error for UE detection.
2791                  */
2792
2793                 if (ue_lo || ue_hi) {
2794                         error_detected = true;
2795                         dev_err(dev,
2796                                 "Unrecoverable Error detected in the adapter");
2797                         dev_err(dev, "Please reboot server to recover");
2798                         if (skyhawk_chip(adapter))
2799                                 adapter->hw_error = true;
2800                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2801                                 if (ue_lo & 1)
2802                                         dev_err(dev, "UE: %s bit set\n",
2803                                                 ue_status_low_desc[i]);
2804                         }
2805                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2806                                 if (ue_hi & 1)
2807                                         dev_err(dev, "UE: %s bit set\n",
2808                                                 ue_status_hi_desc[i]);
2809                         }
2810                 }
2811         }
2812         if (error_detected)
2813                 netif_carrier_off(netdev);
2814 }
2815
2816 static void be_msix_disable(struct be_adapter *adapter)
2817 {
2818         if (msix_enabled(adapter)) {
2819                 pci_disable_msix(adapter->pdev);
2820                 adapter->num_msix_vec = 0;
2821                 adapter->num_msix_roce_vec = 0;
2822         }
2823 }
2824
2825 static int be_msix_enable(struct be_adapter *adapter)
2826 {
2827         int i, num_vec;
2828         struct device *dev = &adapter->pdev->dev;
2829
2830         /* If RoCE is supported, program the max number of NIC vectors that
2831          * may be configured via set-channels, along with vectors needed for
2832          * RoCe. Else, just program the number we'll use initially.
2833          */
2834         if (be_roce_supported(adapter))
2835                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2836                                 2 * num_online_cpus());
2837         else
2838                 num_vec = adapter->cfg_num_qs;
2839
2840         for (i = 0; i < num_vec; i++)
2841                 adapter->msix_entries[i].entry = i;
2842
2843         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2844                                         MIN_MSIX_VECTORS, num_vec);
2845         if (num_vec < 0)
2846                 goto fail;
2847
2848         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2849                 adapter->num_msix_roce_vec = num_vec / 2;
2850                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2851                          adapter->num_msix_roce_vec);
2852         }
2853
2854         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2855
2856         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2857                  adapter->num_msix_vec);
2858         return 0;
2859
2860 fail:
2861         dev_warn(dev, "MSIx enable failed\n");
2862
2863         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2864         if (!be_physfn(adapter))
2865                 return num_vec;
2866         return 0;
2867 }
2868
2869 static inline int be_msix_vec_get(struct be_adapter *adapter,
2870                                   struct be_eq_obj *eqo)
2871 {
2872         return adapter->msix_entries[eqo->msix_idx].vector;
2873 }
2874
2875 static int be_msix_register(struct be_adapter *adapter)
2876 {
2877         struct net_device *netdev = adapter->netdev;
2878         struct be_eq_obj *eqo;
2879         int status, i, vec;
2880
2881         for_all_evt_queues(adapter, eqo, i) {
2882                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2883                 vec = be_msix_vec_get(adapter, eqo);
2884                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2885                 if (status)
2886                         goto err_msix;
2887         }
2888
2889         return 0;
2890 err_msix:
2891         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2892                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2893         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2894                  status);
2895         be_msix_disable(adapter);
2896         return status;
2897 }
2898
2899 static int be_irq_register(struct be_adapter *adapter)
2900 {
2901         struct net_device *netdev = adapter->netdev;
2902         int status;
2903
2904         if (msix_enabled(adapter)) {
2905                 status = be_msix_register(adapter);
2906                 if (status == 0)
2907                         goto done;
2908                 /* INTx is not supported for VF */
2909                 if (!be_physfn(adapter))
2910                         return status;
2911         }
2912
2913         /* INTx: only the first EQ is used */
2914         netdev->irq = adapter->pdev->irq;
2915         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2916                              &adapter->eq_obj[0]);
2917         if (status) {
2918                 dev_err(&adapter->pdev->dev,
2919                         "INTx request IRQ failed - err %d\n", status);
2920                 return status;
2921         }
2922 done:
2923         adapter->isr_registered = true;
2924         return 0;
2925 }
2926
2927 static void be_irq_unregister(struct be_adapter *adapter)
2928 {
2929         struct net_device *netdev = adapter->netdev;
2930         struct be_eq_obj *eqo;
2931         int i;
2932
2933         if (!adapter->isr_registered)
2934                 return;
2935
2936         /* INTx */
2937         if (!msix_enabled(adapter)) {
2938                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2939                 goto done;
2940         }
2941
2942         /* MSIx */
2943         for_all_evt_queues(adapter, eqo, i)
2944                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2945
2946 done:
2947         adapter->isr_registered = false;
2948 }
2949
2950 static void be_rx_qs_destroy(struct be_adapter *adapter)
2951 {
2952         struct be_queue_info *q;
2953         struct be_rx_obj *rxo;
2954         int i;
2955
2956         for_all_rx_queues(adapter, rxo, i) {
2957                 q = &rxo->q;
2958                 if (q->created) {
2959                         be_cmd_rxq_destroy(adapter, q);
2960                         be_rx_cq_clean(rxo);
2961                 }
2962                 be_queue_free(adapter, q);
2963         }
2964 }
2965
2966 static int be_close(struct net_device *netdev)
2967 {
2968         struct be_adapter *adapter = netdev_priv(netdev);
2969         struct be_eq_obj *eqo;
2970         int i;
2971
2972         /* This protection is needed as be_close() may be called even when the
2973          * adapter is in cleared state (after eeh perm failure)
2974          */
2975         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
2976                 return 0;
2977
2978         be_roce_dev_close(adapter);
2979
2980         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
2981                 for_all_evt_queues(adapter, eqo, i) {
2982                         napi_disable(&eqo->napi);
2983                         be_disable_busy_poll(eqo);
2984                 }
2985                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
2986         }
2987
2988         be_async_mcc_disable(adapter);
2989
2990         /* Wait for all pending tx completions to arrive so that
2991          * all tx skbs are freed.
2992          */
2993         netif_tx_disable(netdev);
2994         be_tx_compl_clean(adapter);
2995
2996         be_rx_qs_destroy(adapter);
2997         be_clear_uc_list(adapter);
2998
2999         for_all_evt_queues(adapter, eqo, i) {
3000                 if (msix_enabled(adapter))
3001                         synchronize_irq(be_msix_vec_get(adapter, eqo));
3002                 else
3003                         synchronize_irq(netdev->irq);
3004                 be_eq_clean(eqo);
3005         }
3006
3007         be_irq_unregister(adapter);
3008
3009         return 0;
3010 }
3011
3012 static int be_rx_qs_create(struct be_adapter *adapter)
3013 {
3014         struct rss_info *rss = &adapter->rss_info;
3015         u8 rss_key[RSS_HASH_KEY_LEN];
3016         struct be_rx_obj *rxo;
3017         int rc, i, j;
3018
3019         for_all_rx_queues(adapter, rxo, i) {
3020                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
3021                                     sizeof(struct be_eth_rx_d));
3022                 if (rc)
3023                         return rc;
3024         }
3025
3026         /* The FW would like the default RXQ to be created first */
3027         rxo = default_rxo(adapter);
3028         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
3029                                adapter->if_handle, false, &rxo->rss_id);
3030         if (rc)
3031                 return rc;
3032
3033         for_all_rss_queues(adapter, rxo, i) {
3034                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
3035                                        rx_frag_size, adapter->if_handle,
3036                                        true, &rxo->rss_id);
3037                 if (rc)
3038                         return rc;
3039         }
3040
3041         if (be_multi_rxq(adapter)) {
3042                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
3043                         j += adapter->num_rx_qs - 1) {
3044                         for_all_rss_queues(adapter, rxo, i) {
3045                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
3046                                         break;
3047                                 rss->rsstable[j + i] = rxo->rss_id;
3048                                 rss->rss_queue[j + i] = i;
3049                         }
3050                 }
3051                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
3052                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
3053
3054                 if (!BEx_chip(adapter))
3055                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
3056                                 RSS_ENABLE_UDP_IPV6;
3057         } else {
3058                 /* Disable RSS, if only default RX Q is created */
3059                 rss->rss_flags = RSS_ENABLE_NONE;
3060         }
3061
3062         netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN);
3063         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
3064                                128, rss_key);
3065         if (rc) {
3066                 rss->rss_flags = RSS_ENABLE_NONE;
3067                 return rc;
3068         }
3069
3070         memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN);
3071
3072         /* First time posting */
3073         for_all_rx_queues(adapter, rxo, i)
3074                 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
3075         return 0;
3076 }
3077
3078 static int be_open(struct net_device *netdev)
3079 {
3080         struct be_adapter *adapter = netdev_priv(netdev);
3081         struct be_eq_obj *eqo;
3082         struct be_rx_obj *rxo;
3083         struct be_tx_obj *txo;
3084         u8 link_status;
3085         int status, i;
3086
3087         status = be_rx_qs_create(adapter);
3088         if (status)
3089                 goto err;
3090
3091         status = be_irq_register(adapter);
3092         if (status)
3093                 goto err;
3094
3095         for_all_rx_queues(adapter, rxo, i)
3096                 be_cq_notify(adapter, rxo->cq.id, true, 0);
3097
3098         for_all_tx_queues(adapter, txo, i)
3099                 be_cq_notify(adapter, txo->cq.id, true, 0);
3100
3101         be_async_mcc_enable(adapter);
3102
3103         for_all_evt_queues(adapter, eqo, i) {
3104                 napi_enable(&eqo->napi);
3105                 be_enable_busy_poll(eqo);
3106                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
3107         }
3108         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
3109
3110         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
3111         if (!status)
3112                 be_link_status_update(adapter, link_status);
3113
3114         netif_tx_start_all_queues(netdev);
3115         be_roce_dev_open(adapter);
3116
3117 #ifdef CONFIG_BE2NET_VXLAN
3118         if (skyhawk_chip(adapter))
3119                 vxlan_get_rx_port(netdev);
3120 #endif
3121
3122         return 0;
3123 err:
3124         be_close(adapter->netdev);
3125         return -EIO;
3126 }
3127
3128 static int be_setup_wol(struct be_adapter *adapter, bool enable)
3129 {
3130         struct be_dma_mem cmd;
3131         int status = 0;
3132         u8 mac[ETH_ALEN];
3133
3134         memset(mac, 0, ETH_ALEN);
3135
3136         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
3137         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
3138                                      GFP_KERNEL);
3139         if (!cmd.va)
3140                 return -ENOMEM;
3141
3142         if (enable) {
3143                 status = pci_write_config_dword(adapter->pdev,
3144                                                 PCICFG_PM_CONTROL_OFFSET,
3145                                                 PCICFG_PM_CONTROL_MASK);
3146                 if (status) {
3147                         dev_err(&adapter->pdev->dev,
3148                                 "Could not enable Wake-on-lan\n");
3149                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
3150                                           cmd.dma);
3151                         return status;
3152                 }
3153                 status = be_cmd_enable_magic_wol(adapter,
3154                                                  adapter->netdev->dev_addr,
3155                                                  &cmd);
3156                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
3157                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
3158         } else {
3159                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
3160                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
3161                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
3162         }
3163
3164         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3165         return status;
3166 }
3167
3168 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac)
3169 {
3170         u32 addr;
3171
3172         addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0);
3173
3174         mac[5] = (u8)(addr & 0xFF);
3175         mac[4] = (u8)((addr >> 8) & 0xFF);
3176         mac[3] = (u8)((addr >> 16) & 0xFF);
3177         /* Use the OUI from the current MAC address */
3178         memcpy(mac, adapter->netdev->dev_addr, 3);
3179 }
3180
3181 /*
3182  * Generate a seed MAC address from the PF MAC Address using jhash.
3183  * MAC Address for VFs are assigned incrementally starting from the seed.
3184  * These addresses are programmed in the ASIC by the PF and the VF driver
3185  * queries for the MAC address during its probe.
3186  */
3187 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3188 {
3189         u32 vf;
3190         int status = 0;
3191         u8 mac[ETH_ALEN];
3192         struct be_vf_cfg *vf_cfg;
3193
3194         be_vf_eth_addr_generate(adapter, mac);
3195
3196         for_all_vfs(adapter, vf_cfg, vf) {
3197                 if (BEx_chip(adapter))
3198                         status = be_cmd_pmac_add(adapter, mac,
3199                                                  vf_cfg->if_handle,
3200                                                  &vf_cfg->pmac_id, vf + 1);
3201                 else
3202                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3203                                                 vf + 1);
3204
3205                 if (status)
3206                         dev_err(&adapter->pdev->dev,
3207                                 "Mac address assignment failed for VF %d\n",
3208                                 vf);
3209                 else
3210                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3211
3212                 mac[5] += 1;
3213         }
3214         return status;
3215 }
3216
3217 static int be_vfs_mac_query(struct be_adapter *adapter)
3218 {
3219         int status, vf;
3220         u8 mac[ETH_ALEN];
3221         struct be_vf_cfg *vf_cfg;
3222
3223         for_all_vfs(adapter, vf_cfg, vf) {
3224                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3225                                                mac, vf_cfg->if_handle,
3226                                                false, vf+1);
3227                 if (status)
3228                         return status;
3229                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3230         }
3231         return 0;
3232 }
3233
3234 static void be_vf_clear(struct be_adapter *adapter)
3235 {
3236         struct be_vf_cfg *vf_cfg;
3237         u32 vf;
3238
3239         if (pci_vfs_assigned(adapter->pdev)) {
3240                 dev_warn(&adapter->pdev->dev,
3241                          "VFs are assigned to VMs: not disabling VFs\n");
3242                 goto done;
3243         }
3244
3245         pci_disable_sriov(adapter->pdev);
3246
3247         for_all_vfs(adapter, vf_cfg, vf) {
3248                 if (BEx_chip(adapter))
3249                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3250                                         vf_cfg->pmac_id, vf + 1);
3251                 else
3252                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3253                                        vf + 1);
3254
3255                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3256         }
3257 done:
3258         kfree(adapter->vf_cfg);
3259         adapter->num_vfs = 0;
3260         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3261 }
3262
3263 static void be_clear_queues(struct be_adapter *adapter)
3264 {
3265         be_mcc_queues_destroy(adapter);
3266         be_rx_cqs_destroy(adapter);
3267         be_tx_queues_destroy(adapter);
3268         be_evt_queues_destroy(adapter);
3269 }
3270
3271 static void be_cancel_worker(struct be_adapter *adapter)
3272 {
3273         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3274                 cancel_delayed_work_sync(&adapter->work);
3275                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3276         }
3277 }
3278
3279 static void be_mac_clear(struct be_adapter *adapter)
3280 {
3281         if (adapter->pmac_id) {
3282                 be_cmd_pmac_del(adapter, adapter->if_handle,
3283                                 adapter->pmac_id[0], 0);
3284                 kfree(adapter->pmac_id);
3285                 adapter->pmac_id = NULL;
3286         }
3287 }
3288
3289 #ifdef CONFIG_BE2NET_VXLAN
3290 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3291 {
3292         struct net_device *netdev = adapter->netdev;
3293
3294         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3295                 be_cmd_manage_iface(adapter, adapter->if_handle,
3296                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3297
3298         if (adapter->vxlan_port)
3299                 be_cmd_set_vxlan_port(adapter, 0);
3300
3301         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3302         adapter->vxlan_port = 0;
3303
3304         netdev->hw_enc_features = 0;
3305         netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3306         netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3307 }
3308 #endif
3309
3310 static int be_clear(struct be_adapter *adapter)
3311 {
3312         be_cancel_worker(adapter);
3313
3314         if (sriov_enabled(adapter))
3315                 be_vf_clear(adapter);
3316
3317         /* Re-configure FW to distribute resources evenly across max-supported
3318          * number of VFs, only when VFs are not already enabled.
3319          */
3320         if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev))
3321                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3322                                         pci_sriov_get_totalvfs(adapter->pdev));
3323
3324 #ifdef CONFIG_BE2NET_VXLAN
3325         be_disable_vxlan_offloads(adapter);
3326 #endif
3327         /* delete the primary mac along with the uc-mac list */
3328         be_mac_clear(adapter);
3329
3330         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3331
3332         be_clear_queues(adapter);
3333
3334         be_msix_disable(adapter);
3335         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3336         return 0;
3337 }
3338
3339 static int be_if_create(struct be_adapter *adapter, u32 *if_handle,
3340                         u32 cap_flags, u32 vf)
3341 {
3342         u32 en_flags;
3343         int status;
3344
3345         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3346                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS |
3347                    BE_IF_FLAGS_RSS;
3348
3349         en_flags &= cap_flags;
3350
3351         status = be_cmd_if_create(adapter, cap_flags, en_flags,
3352                                   if_handle, vf);
3353
3354         return status;
3355 }
3356
3357 static int be_vfs_if_create(struct be_adapter *adapter)
3358 {
3359         struct be_resources res = {0};
3360         struct be_vf_cfg *vf_cfg;
3361         u32 cap_flags, vf;
3362         int status;
3363
3364         /* If a FW profile exists, then cap_flags are updated */
3365         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3366                     BE_IF_FLAGS_MULTICAST;
3367
3368         for_all_vfs(adapter, vf_cfg, vf) {
3369                 if (!BE3_chip(adapter)) {
3370                         status = be_cmd_get_profile_config(adapter, &res,
3371                                                            vf + 1);
3372                         if (!status)
3373                                 cap_flags = res.if_cap_flags;
3374                 }
3375
3376                 status = be_if_create(adapter, &vf_cfg->if_handle,
3377                                       cap_flags, vf + 1);
3378                 if (status)
3379                         return status;
3380         }
3381
3382         return 0;
3383 }
3384
3385 static int be_vf_setup_init(struct be_adapter *adapter)
3386 {
3387         struct be_vf_cfg *vf_cfg;
3388         int vf;
3389
3390         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3391                                   GFP_KERNEL);
3392         if (!adapter->vf_cfg)
3393                 return -ENOMEM;
3394
3395         for_all_vfs(adapter, vf_cfg, vf) {
3396                 vf_cfg->if_handle = -1;
3397                 vf_cfg->pmac_id = -1;
3398         }
3399         return 0;
3400 }
3401
3402 static int be_vf_setup(struct be_adapter *adapter)
3403 {
3404         struct device *dev = &adapter->pdev->dev;
3405         struct be_vf_cfg *vf_cfg;
3406         int status, old_vfs, vf;
3407         u32 privileges;
3408
3409         old_vfs = pci_num_vf(adapter->pdev);
3410
3411         status = be_vf_setup_init(adapter);
3412         if (status)
3413                 goto err;
3414
3415         if (old_vfs) {
3416                 for_all_vfs(adapter, vf_cfg, vf) {
3417                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3418                         if (status)
3419                                 goto err;
3420                 }
3421
3422                 status = be_vfs_mac_query(adapter);
3423                 if (status)
3424                         goto err;
3425         } else {
3426                 status = be_vfs_if_create(adapter);
3427                 if (status)
3428                         goto err;
3429
3430                 status = be_vf_eth_addr_config(adapter);
3431                 if (status)
3432                         goto err;
3433         }
3434
3435         for_all_vfs(adapter, vf_cfg, vf) {
3436                 /* Allow VFs to programs MAC/VLAN filters */
3437                 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1);
3438                 if (!status && !(privileges & BE_PRIV_FILTMGMT)) {
3439                         status = be_cmd_set_fn_privileges(adapter,
3440                                                           privileges |
3441                                                           BE_PRIV_FILTMGMT,
3442                                                           vf + 1);
3443                         if (!status)
3444                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3445                                          vf);
3446                 }
3447
3448                 /* Allow full available bandwidth */
3449                 if (!old_vfs)
3450                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3451
3452                 if (!old_vfs) {
3453                         be_cmd_enable_vf(adapter, vf + 1);
3454                         be_cmd_set_logical_link_config(adapter,
3455                                                        IFLA_VF_LINK_STATE_AUTO,
3456                                                        vf+1);
3457                 }
3458         }
3459
3460         if (!old_vfs) {
3461                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3462                 if (status) {
3463                         dev_err(dev, "SRIOV enable failed\n");
3464                         adapter->num_vfs = 0;
3465                         goto err;
3466                 }
3467         }
3468
3469         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3470         return 0;
3471 err:
3472         dev_err(dev, "VF setup failed\n");
3473         be_vf_clear(adapter);
3474         return status;
3475 }
3476
3477 /* Converting function_mode bits on BE3 to SH mc_type enums */
3478
3479 static u8 be_convert_mc_type(u32 function_mode)
3480 {
3481         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3482                 return vNIC1;
3483         else if (function_mode & QNQ_MODE)
3484                 return FLEX10;
3485         else if (function_mode & VNIC_MODE)
3486                 return vNIC2;
3487         else if (function_mode & UMC_ENABLED)
3488                 return UMC;
3489         else
3490                 return MC_NONE;
3491 }
3492
3493 /* On BE2/BE3 FW does not suggest the supported limits */
3494 static void BEx_get_resources(struct be_adapter *adapter,
3495                               struct be_resources *res)
3496 {
3497         bool use_sriov = adapter->num_vfs ? 1 : 0;
3498
3499         if (be_physfn(adapter))
3500                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3501         else
3502                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3503
3504         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3505
3506         if (be_is_mc(adapter)) {
3507                 /* Assuming that there are 4 channels per port,
3508                  * when multi-channel is enabled
3509                  */
3510                 if (be_is_qnq_mode(adapter))
3511                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3512                 else
3513                         /* In a non-qnq multichannel mode, the pvid
3514                          * takes up one vlan entry
3515                          */
3516                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3517         } else {
3518                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3519         }
3520
3521         res->max_mcast_mac = BE_MAX_MC;
3522
3523         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3524          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3525          *    *only* if it is RSS-capable.
3526          */
3527         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3528             !be_physfn(adapter) || (be_is_mc(adapter) &&
3529             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
3530                 res->max_tx_qs = 1;
3531         } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
3532                 struct be_resources super_nic_res = {0};
3533
3534                 /* On a SuperNIC profile, the driver needs to use the
3535                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
3536                  */
3537                 be_cmd_get_profile_config(adapter, &super_nic_res, 0);
3538                 /* Some old versions of BE3 FW don't report max_tx_qs value */
3539                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
3540         } else {
3541                 res->max_tx_qs = BE3_MAX_TX_QS;
3542         }
3543
3544         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3545             !use_sriov && be_physfn(adapter))
3546                 res->max_rss_qs = (adapter->be3_native) ?
3547                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3548         res->max_rx_qs = res->max_rss_qs + 1;
3549
3550         if (be_physfn(adapter))
3551                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3552                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3553         else
3554                 res->max_evt_qs = 1;
3555
3556         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3557         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3558                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3559 }
3560
3561 static void be_setup_init(struct be_adapter *adapter)
3562 {
3563         adapter->vlan_prio_bmap = 0xff;
3564         adapter->phy.link_speed = -1;
3565         adapter->if_handle = -1;
3566         adapter->be3_native = false;
3567         adapter->if_flags = 0;
3568         if (be_physfn(adapter))
3569                 adapter->cmd_privileges = MAX_PRIVILEGES;
3570         else
3571                 adapter->cmd_privileges = MIN_PRIVILEGES;
3572 }
3573
3574 static int be_get_sriov_config(struct be_adapter *adapter)
3575 {
3576         struct device *dev = &adapter->pdev->dev;
3577         struct be_resources res = {0};
3578         int max_vfs, old_vfs;
3579
3580         /* Some old versions of BE3 FW don't report max_vfs value */
3581         be_cmd_get_profile_config(adapter, &res, 0);
3582
3583         if (BE3_chip(adapter) && !res.max_vfs) {
3584                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3585                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3586         }
3587
3588         adapter->pool_res = res;
3589
3590         if (!be_max_vfs(adapter)) {
3591                 if (num_vfs)
3592                         dev_warn(dev, "SRIOV is disabled. Ignoring num_vfs\n");
3593                 adapter->num_vfs = 0;
3594                 return 0;
3595         }
3596
3597         pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3598
3599         /* validate num_vfs module param */
3600         old_vfs = pci_num_vf(adapter->pdev);
3601         if (old_vfs) {
3602                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3603                 if (old_vfs != num_vfs)
3604                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3605                 adapter->num_vfs = old_vfs;
3606         } else {
3607                 if (num_vfs > be_max_vfs(adapter)) {
3608                         dev_info(dev, "Resources unavailable to init %d VFs\n",
3609                                  num_vfs);
3610                         dev_info(dev, "Limiting to %d VFs\n",
3611                                  be_max_vfs(adapter));
3612                 }
3613                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3614         }
3615
3616         return 0;
3617 }
3618
3619 static int be_get_resources(struct be_adapter *adapter)
3620 {
3621         struct device *dev = &adapter->pdev->dev;
3622         struct be_resources res = {0};
3623         int status;
3624
3625         if (BEx_chip(adapter)) {
3626                 BEx_get_resources(adapter, &res);
3627                 adapter->res = res;
3628         }
3629
3630         /* For Lancer, SH etc read per-function resource limits from FW.
3631          * GET_FUNC_CONFIG returns per function guaranteed limits.
3632          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3633          */
3634         if (!BEx_chip(adapter)) {
3635                 status = be_cmd_get_func_config(adapter, &res);
3636                 if (status)
3637                         return status;
3638
3639                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3640                 if (be_roce_supported(adapter))
3641                         res.max_evt_qs /= 2;
3642                 adapter->res = res;
3643         }
3644
3645         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3646                  be_max_txqs(adapter), be_max_rxqs(adapter),
3647                  be_max_rss(adapter), be_max_eqs(adapter),
3648                  be_max_vfs(adapter));
3649         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3650                  be_max_uc(adapter), be_max_mc(adapter),
3651                  be_max_vlans(adapter));
3652
3653         return 0;
3654 }
3655
3656 static void be_sriov_config(struct be_adapter *adapter)
3657 {
3658         struct device *dev = &adapter->pdev->dev;
3659         int status;
3660
3661         status = be_get_sriov_config(adapter);
3662         if (status) {
3663                 dev_err(dev, "Failed to query SR-IOV configuration\n");
3664                 dev_err(dev, "SR-IOV cannot be enabled\n");
3665                 return;
3666         }
3667
3668         /* When the HW is in SRIOV capable configuration, the PF-pool
3669          * resources are equally distributed across the max-number of
3670          * VFs. The user may request only a subset of the max-vfs to be
3671          * enabled. Based on num_vfs, redistribute the resources across
3672          * num_vfs so that each VF will have access to more number of
3673          * resources. This facility is not available in BE3 FW.
3674          * Also, this is done by FW in Lancer chip.
3675          */
3676         if (be_max_vfs(adapter) && !pci_num_vf(adapter->pdev)) {
3677                 status = be_cmd_set_sriov_config(adapter,
3678                                                  adapter->pool_res,
3679                                                  adapter->num_vfs);
3680                 if (status)
3681                         dev_err(dev, "Failed to optimize SR-IOV resources\n");
3682         }
3683 }
3684
3685 static int be_get_config(struct be_adapter *adapter)
3686 {
3687         u16 profile_id;
3688         int status;
3689
3690         status = be_cmd_query_fw_cfg(adapter);
3691         if (status)
3692                 return status;
3693
3694         be_cmd_query_port_name(adapter);
3695
3696         if (be_physfn(adapter)) {
3697                 status = be_cmd_get_active_profile(adapter, &profile_id);
3698                 if (!status)
3699                         dev_info(&adapter->pdev->dev,
3700                                  "Using profile 0x%x\n", profile_id);
3701         }
3702
3703         if (!BE2_chip(adapter) && be_physfn(adapter))
3704                 be_sriov_config(adapter);
3705
3706         status = be_get_resources(adapter);
3707         if (status)
3708                 return status;
3709
3710         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3711                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3712         if (!adapter->pmac_id)
3713                 return -ENOMEM;
3714
3715         /* Sanitize cfg_num_qs based on HW and platform limits */
3716         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3717
3718         return 0;
3719 }
3720
3721 static int be_mac_setup(struct be_adapter *adapter)
3722 {
3723         u8 mac[ETH_ALEN];
3724         int status;
3725
3726         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3727                 status = be_cmd_get_perm_mac(adapter, mac);
3728                 if (status)
3729                         return status;
3730
3731                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3732                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3733         } else {
3734                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3735                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3736         }
3737
3738         /* For BE3-R VFs, the PF programs the initial MAC address */
3739         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3740                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3741                                 &adapter->pmac_id[0], 0);
3742         return 0;
3743 }
3744
3745 static void be_schedule_worker(struct be_adapter *adapter)
3746 {
3747         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3748         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3749 }
3750
3751 static int be_setup_queues(struct be_adapter *adapter)
3752 {
3753         struct net_device *netdev = adapter->netdev;
3754         int status;
3755
3756         status = be_evt_queues_create(adapter);
3757         if (status)
3758                 goto err;
3759
3760         status = be_tx_qs_create(adapter);
3761         if (status)
3762                 goto err;
3763
3764         status = be_rx_cqs_create(adapter);
3765         if (status)
3766                 goto err;
3767
3768         status = be_mcc_queues_create(adapter);
3769         if (status)
3770                 goto err;
3771
3772         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3773         if (status)
3774                 goto err;
3775
3776         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3777         if (status)
3778                 goto err;
3779
3780         return 0;
3781 err:
3782         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3783         return status;
3784 }
3785
3786 int be_update_queues(struct be_adapter *adapter)
3787 {
3788         struct net_device *netdev = adapter->netdev;
3789         int status;
3790
3791         if (netif_running(netdev))
3792                 be_close(netdev);
3793
3794         be_cancel_worker(adapter);
3795
3796         /* If any vectors have been shared with RoCE we cannot re-program
3797          * the MSIx table.
3798          */
3799         if (!adapter->num_msix_roce_vec)
3800                 be_msix_disable(adapter);
3801
3802         be_clear_queues(adapter);
3803
3804         if (!msix_enabled(adapter)) {
3805                 status = be_msix_enable(adapter);
3806                 if (status)
3807                         return status;
3808         }
3809
3810         status = be_setup_queues(adapter);
3811         if (status)
3812                 return status;
3813
3814         be_schedule_worker(adapter);
3815
3816         if (netif_running(netdev))
3817                 status = be_open(netdev);
3818
3819         return status;
3820 }
3821
3822 static inline int fw_major_num(const char *fw_ver)
3823 {
3824         int fw_major = 0, i;
3825
3826         i = sscanf(fw_ver, "%d.", &fw_major);
3827         if (i != 1)
3828                 return 0;
3829
3830         return fw_major;
3831 }
3832
3833 static int be_setup(struct be_adapter *adapter)
3834 {
3835         struct device *dev = &adapter->pdev->dev;
3836         int status;
3837
3838         be_setup_init(adapter);
3839
3840         if (!lancer_chip(adapter))
3841                 be_cmd_req_native_mode(adapter);
3842
3843         status = be_get_config(adapter);
3844         if (status)
3845                 goto err;
3846
3847         status = be_msix_enable(adapter);
3848         if (status)
3849                 goto err;
3850
3851         status = be_if_create(adapter, &adapter->if_handle,
3852                               be_if_cap_flags(adapter), 0);
3853         if (status)
3854                 goto err;
3855
3856         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3857         rtnl_lock();
3858         status = be_setup_queues(adapter);
3859         rtnl_unlock();
3860         if (status)
3861                 goto err;
3862
3863         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3864
3865         status = be_mac_setup(adapter);
3866         if (status)
3867                 goto err;
3868
3869         be_cmd_get_fw_ver(adapter);
3870         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
3871
3872         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3873                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
3874                         adapter->fw_ver);
3875                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3876         }
3877
3878         if (adapter->vlans_added)
3879                 be_vid_config(adapter);
3880
3881         be_set_rx_mode(adapter->netdev);
3882
3883         be_cmd_get_acpi_wol_cap(adapter);
3884
3885         status = be_cmd_set_flow_control(adapter, adapter->tx_fc,
3886                                          adapter->rx_fc);
3887         if (status)
3888                 be_cmd_get_flow_control(adapter, &adapter->tx_fc,
3889                                         &adapter->rx_fc);
3890
3891         dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n",
3892                  adapter->tx_fc, adapter->rx_fc);
3893
3894         if (be_physfn(adapter))
3895                 be_cmd_set_logical_link_config(adapter,
3896                                                IFLA_VF_LINK_STATE_AUTO, 0);
3897
3898         if (adapter->num_vfs)
3899                 be_vf_setup(adapter);
3900
3901         status = be_cmd_get_phy_info(adapter);
3902         if (!status && be_pause_supported(adapter))
3903                 adapter->phy.fc_autoneg = 1;
3904
3905         be_schedule_worker(adapter);
3906         adapter->flags |= BE_FLAGS_SETUP_DONE;
3907         return 0;
3908 err:
3909         be_clear(adapter);
3910         return status;
3911 }
3912
3913 #ifdef CONFIG_NET_POLL_CONTROLLER
3914 static void be_netpoll(struct net_device *netdev)
3915 {
3916         struct be_adapter *adapter = netdev_priv(netdev);
3917         struct be_eq_obj *eqo;
3918         int i;
3919
3920         for_all_evt_queues(adapter, eqo, i) {
3921                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3922                 napi_schedule(&eqo->napi);
3923         }
3924 }
3925 #endif
3926
3927 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
3928
3929 static bool phy_flashing_required(struct be_adapter *adapter)
3930 {
3931         return (adapter->phy.phy_type == PHY_TYPE_TN_8022 &&
3932                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3933 }
3934
3935 static bool is_comp_in_ufi(struct be_adapter *adapter,
3936                            struct flash_section_info *fsec, int type)
3937 {
3938         int i = 0, img_type = 0;
3939         struct flash_section_info_g2 *fsec_g2 = NULL;
3940
3941         if (BE2_chip(adapter))
3942                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3943
3944         for (i = 0; i < MAX_FLASH_COMP; i++) {
3945                 if (fsec_g2)
3946                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
3947                 else
3948                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
3949
3950                 if (img_type == type)
3951                         return true;
3952         }
3953         return false;
3954
3955 }
3956
3957 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
3958                                                 int header_size,
3959                                                 const struct firmware *fw)
3960 {
3961         struct flash_section_info *fsec = NULL;
3962         const u8 *p = fw->data;
3963
3964         p += header_size;
3965         while (p < (fw->data + fw->size)) {
3966                 fsec = (struct flash_section_info *)p;
3967                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
3968                         return fsec;
3969                 p += 32;
3970         }
3971         return NULL;
3972 }
3973
3974 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
3975                               u32 img_offset, u32 img_size, int hdr_size,
3976                               u16 img_optype, bool *crc_match)
3977 {
3978         u32 crc_offset;
3979         int status;
3980         u8 crc[4];
3981
3982         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_offset,
3983                                       img_size - 4);
3984         if (status)
3985                 return status;
3986
3987         crc_offset = hdr_size + img_offset + img_size - 4;
3988
3989         /* Skip flashing, if crc of flashed region matches */
3990         if (!memcmp(crc, p + crc_offset, 4))
3991                 *crc_match = true;
3992         else
3993                 *crc_match = false;
3994
3995         return status;
3996 }
3997
3998 static int be_flash(struct be_adapter *adapter, const u8 *img,
3999                     struct be_dma_mem *flash_cmd, int optype, int img_size,
4000                     u32 img_offset)
4001 {
4002         u32 flash_op, num_bytes, total_bytes = img_size, bytes_sent = 0;
4003         struct be_cmd_write_flashrom *req = flash_cmd->va;
4004         int status;
4005
4006         while (total_bytes) {
4007                 num_bytes = min_t(u32, 32*1024, total_bytes);
4008
4009                 total_bytes -= num_bytes;
4010
4011                 if (!total_bytes) {
4012                         if (optype == OPTYPE_PHY_FW)
4013                                 flash_op = FLASHROM_OPER_PHY_FLASH;
4014                         else
4015                                 flash_op = FLASHROM_OPER_FLASH;
4016                 } else {
4017                         if (optype == OPTYPE_PHY_FW)
4018                                 flash_op = FLASHROM_OPER_PHY_SAVE;
4019                         else
4020                                 flash_op = FLASHROM_OPER_SAVE;
4021                 }
4022
4023                 memcpy(req->data_buf, img, num_bytes);
4024                 img += num_bytes;
4025                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
4026                                                flash_op, img_offset +
4027                                                bytes_sent, num_bytes);
4028                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
4029                     optype == OPTYPE_PHY_FW)
4030                         break;
4031                 else if (status)
4032                         return status;
4033
4034                 bytes_sent += num_bytes;
4035         }
4036         return 0;
4037 }
4038
4039 /* For BE2, BE3 and BE3-R */
4040 static int be_flash_BEx(struct be_adapter *adapter,
4041                         const struct firmware *fw,
4042                         struct be_dma_mem *flash_cmd, int num_of_images)
4043 {
4044         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
4045         struct device *dev = &adapter->pdev->dev;
4046         struct flash_section_info *fsec = NULL;
4047         int status, i, filehdr_size, num_comp;
4048         const struct flash_comp *pflashcomp;
4049         bool crc_match;
4050         const u8 *p;
4051
4052         struct flash_comp gen3_flash_types[] = {
4053                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
4054                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
4055                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
4056                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
4057                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
4058                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
4059                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
4060                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
4061                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
4062                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
4063                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
4064                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
4065                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
4066                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
4067                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
4068                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
4069                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
4070                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
4071                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
4072                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
4073         };
4074
4075         struct flash_comp gen2_flash_types[] = {
4076                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
4077                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
4078                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
4079                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
4080                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
4081                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
4082                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
4083                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
4084                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
4085                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
4086                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
4087                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
4088                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
4089                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
4090                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
4091                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
4092         };
4093
4094         if (BE3_chip(adapter)) {
4095                 pflashcomp = gen3_flash_types;
4096                 filehdr_size = sizeof(struct flash_file_hdr_g3);
4097                 num_comp = ARRAY_SIZE(gen3_flash_types);
4098         } else {
4099                 pflashcomp = gen2_flash_types;
4100                 filehdr_size = sizeof(struct flash_file_hdr_g2);
4101                 num_comp = ARRAY_SIZE(gen2_flash_types);
4102                 img_hdrs_size = 0;
4103         }
4104
4105         /* Get flash section info*/
4106         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4107         if (!fsec) {
4108                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4109                 return -1;
4110         }
4111         for (i = 0; i < num_comp; i++) {
4112                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
4113                         continue;
4114
4115                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
4116                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
4117                         continue;
4118
4119                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
4120                     !phy_flashing_required(adapter))
4121                                 continue;
4122
4123                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
4124                         status = be_check_flash_crc(adapter, fw->data,
4125                                                     pflashcomp[i].offset,
4126                                                     pflashcomp[i].size,
4127                                                     filehdr_size +
4128                                                     img_hdrs_size,
4129                                                     OPTYPE_REDBOOT, &crc_match);
4130                         if (status) {
4131                                 dev_err(dev,
4132                                         "Could not get CRC for 0x%x region\n",
4133                                         pflashcomp[i].optype);
4134                                 continue;
4135                         }
4136
4137                         if (crc_match)
4138                                 continue;
4139                 }
4140
4141                 p = fw->data + filehdr_size + pflashcomp[i].offset +
4142                         img_hdrs_size;
4143                 if (p + pflashcomp[i].size > fw->data + fw->size)
4144                         return -1;
4145
4146                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
4147                                   pflashcomp[i].size, 0);
4148                 if (status) {
4149                         dev_err(dev, "Flashing section type 0x%x failed\n",
4150                                 pflashcomp[i].img_type);
4151                         return status;
4152                 }
4153         }
4154         return 0;
4155 }
4156
4157 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
4158 {
4159         u32 img_type = le32_to_cpu(fsec_entry.type);
4160         u16 img_optype = le16_to_cpu(fsec_entry.optype);
4161
4162         if (img_optype != 0xFFFF)
4163                 return img_optype;
4164
4165         switch (img_type) {
4166         case IMAGE_FIRMWARE_iSCSI:
4167                 img_optype = OPTYPE_ISCSI_ACTIVE;
4168                 break;
4169         case IMAGE_BOOT_CODE:
4170                 img_optype = OPTYPE_REDBOOT;
4171                 break;
4172         case IMAGE_OPTION_ROM_ISCSI:
4173                 img_optype = OPTYPE_BIOS;
4174                 break;
4175         case IMAGE_OPTION_ROM_PXE:
4176                 img_optype = OPTYPE_PXE_BIOS;
4177                 break;
4178         case IMAGE_OPTION_ROM_FCoE:
4179                 img_optype = OPTYPE_FCOE_BIOS;
4180                 break;
4181         case IMAGE_FIRMWARE_BACKUP_iSCSI:
4182                 img_optype = OPTYPE_ISCSI_BACKUP;
4183                 break;
4184         case IMAGE_NCSI:
4185                 img_optype = OPTYPE_NCSI_FW;
4186                 break;
4187         case IMAGE_FLASHISM_JUMPVECTOR:
4188                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
4189                 break;
4190         case IMAGE_FIRMWARE_PHY:
4191                 img_optype = OPTYPE_SH_PHY_FW;
4192                 break;
4193         case IMAGE_REDBOOT_DIR:
4194                 img_optype = OPTYPE_REDBOOT_DIR;
4195                 break;
4196         case IMAGE_REDBOOT_CONFIG:
4197                 img_optype = OPTYPE_REDBOOT_CONFIG;
4198                 break;
4199         case IMAGE_UFI_DIR:
4200                 img_optype = OPTYPE_UFI_DIR;
4201                 break;
4202         default:
4203                 break;
4204         }
4205
4206         return img_optype;
4207 }
4208
4209 static int be_flash_skyhawk(struct be_adapter *adapter,
4210                             const struct firmware *fw,
4211                             struct be_dma_mem *flash_cmd, int num_of_images)
4212 {
4213         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
4214         bool crc_match, old_fw_img, flash_offset_support = true;
4215         struct device *dev = &adapter->pdev->dev;
4216         struct flash_section_info *fsec = NULL;
4217         u32 img_offset, img_size, img_type;
4218         u16 img_optype, flash_optype;
4219         int status, i, filehdr_size;
4220         const u8 *p;
4221
4222         filehdr_size = sizeof(struct flash_file_hdr_g3);
4223         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4224         if (!fsec) {
4225                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4226                 return -EINVAL;
4227         }
4228
4229 retry_flash:
4230         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4231                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4232                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4233                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4234                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4235                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4236
4237                 if (img_optype == 0xFFFF)
4238                         continue;
4239
4240                 if (flash_offset_support)
4241                         flash_optype = OPTYPE_OFFSET_SPECIFIED;
4242                 else
4243                         flash_optype = img_optype;
4244
4245                 /* Don't bother verifying CRC if an old FW image is being
4246                  * flashed
4247                  */
4248                 if (old_fw_img)
4249                         goto flash;
4250
4251                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4252                                             img_size, filehdr_size +
4253                                             img_hdrs_size, flash_optype,
4254                                             &crc_match);
4255                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4256                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4257                         /* The current FW image on the card does not support
4258                          * OFFSET based flashing. Retry using older mechanism
4259                          * of OPTYPE based flashing
4260                          */
4261                         if (flash_optype == OPTYPE_OFFSET_SPECIFIED) {
4262                                 flash_offset_support = false;
4263                                 goto retry_flash;
4264                         }
4265
4266                         /* The current FW image on the card does not recognize
4267                          * the new FLASH op_type. The FW download is partially
4268                          * complete. Reboot the server now to enable FW image
4269                          * to recognize the new FLASH op_type. To complete the
4270                          * remaining process, download the same FW again after
4271                          * the reboot.
4272                          */
4273                         dev_err(dev, "Flash incomplete. Reset the server\n");
4274                         dev_err(dev, "Download FW image again after reset\n");
4275                         return -EAGAIN;
4276                 } else if (status) {
4277                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4278                                 img_optype);
4279                         return -EFAULT;
4280                 }
4281
4282                 if (crc_match)
4283                         continue;
4284
4285 flash:
4286                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4287                 if (p + img_size > fw->data + fw->size)
4288                         return -1;
4289
4290                 status = be_flash(adapter, p, flash_cmd, flash_optype, img_size,
4291                                   img_offset);
4292
4293                 /* The current FW image on the card does not support OFFSET
4294                  * based flashing. Retry using older mechanism of OPTYPE based
4295                  * flashing
4296                  */
4297                 if (base_status(status) == MCC_STATUS_ILLEGAL_FIELD &&
4298                     flash_optype == OPTYPE_OFFSET_SPECIFIED) {
4299                         flash_offset_support = false;
4300                         goto retry_flash;
4301                 }
4302
4303                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4304                  * UFI_DIR region
4305                  */
4306                 if (old_fw_img &&
4307                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4308                      (img_optype == OPTYPE_UFI_DIR &&
4309                       base_status(status) == MCC_STATUS_FAILED))) {
4310                         continue;
4311                 } else if (status) {
4312                         dev_err(dev, "Flashing section type 0x%x failed\n",
4313                                 img_type);
4314                         return -EFAULT;
4315                 }
4316         }
4317         return 0;
4318 }
4319
4320 static int lancer_fw_download(struct be_adapter *adapter,
4321                               const struct firmware *fw)
4322 {
4323 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4324 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4325         struct device *dev = &adapter->pdev->dev;
4326         struct be_dma_mem flash_cmd;
4327         const u8 *data_ptr = NULL;
4328         u8 *dest_image_ptr = NULL;
4329         size_t image_size = 0;
4330         u32 chunk_size = 0;
4331         u32 data_written = 0;
4332         u32 offset = 0;
4333         int status = 0;
4334         u8 add_status = 0;
4335         u8 change_status;
4336
4337         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4338                 dev_err(dev, "FW image size should be multiple of 4\n");
4339                 return -EINVAL;
4340         }
4341
4342         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4343                                 + LANCER_FW_DOWNLOAD_CHUNK;
4344         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size,
4345                                           &flash_cmd.dma, GFP_KERNEL);
4346         if (!flash_cmd.va)
4347                 return -ENOMEM;
4348
4349         dest_image_ptr = flash_cmd.va +
4350                                 sizeof(struct lancer_cmd_req_write_object);
4351         image_size = fw->size;
4352         data_ptr = fw->data;
4353
4354         while (image_size) {
4355                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4356
4357                 /* Copy the image chunk content. */
4358                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4359
4360                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4361                                                  chunk_size, offset,
4362                                                  LANCER_FW_DOWNLOAD_LOCATION,
4363                                                  &data_written, &change_status,
4364                                                  &add_status);
4365                 if (status)
4366                         break;
4367
4368                 offset += data_written;
4369                 data_ptr += data_written;
4370                 image_size -= data_written;
4371         }
4372
4373         if (!status) {
4374                 /* Commit the FW written */
4375                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4376                                                  0, offset,
4377                                                  LANCER_FW_DOWNLOAD_LOCATION,
4378                                                  &data_written, &change_status,
4379                                                  &add_status);
4380         }
4381
4382         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4383         if (status) {
4384                 dev_err(dev, "Firmware load error\n");
4385                 return be_cmd_status(status);
4386         }
4387
4388         dev_info(dev, "Firmware flashed successfully\n");
4389
4390         if (change_status == LANCER_FW_RESET_NEEDED) {
4391                 dev_info(dev, "Resetting adapter to activate new FW\n");
4392                 status = lancer_physdev_ctrl(adapter,
4393                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4394                 if (status) {
4395                         dev_err(dev, "Adapter busy, could not reset FW\n");
4396                         dev_err(dev, "Reboot server to activate new FW\n");
4397                 }
4398         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4399                 dev_info(dev, "Reboot server to activate new FW\n");
4400         }
4401
4402         return 0;
4403 }
4404
4405 #define BE2_UFI         2
4406 #define BE3_UFI         3
4407 #define BE3R_UFI        10
4408 #define SH_UFI          4
4409 #define SH_P2_UFI       11
4410
4411 static int be_get_ufi_type(struct be_adapter *adapter,
4412                            struct flash_file_hdr_g3 *fhdr)
4413 {
4414         if (!fhdr) {
4415                 dev_err(&adapter->pdev->dev, "Invalid FW UFI file");
4416                 return -1;
4417         }
4418
4419         /* First letter of the build version is used to identify
4420          * which chip this image file is meant for.
4421          */
4422         switch (fhdr->build[0]) {
4423         case BLD_STR_UFI_TYPE_SH:
4424                 return (fhdr->asic_type_rev == ASIC_REV_P2) ? SH_P2_UFI :
4425                                                                 SH_UFI;
4426         case BLD_STR_UFI_TYPE_BE3:
4427                 return (fhdr->asic_type_rev == ASIC_REV_B0) ? BE3R_UFI :
4428                                                                 BE3_UFI;
4429         case BLD_STR_UFI_TYPE_BE2:
4430                 return BE2_UFI;
4431         default:
4432                 return -1;
4433         }
4434 }
4435
4436 /* Check if the flash image file is compatible with the adapter that
4437  * is being flashed.
4438  * BE3 chips with asic-rev B0 must be flashed only with BE3R_UFI type.
4439  * Skyhawk chips with asic-rev P2 must be flashed only with SH_P2_UFI type.
4440  */
4441 static bool be_check_ufi_compatibility(struct be_adapter *adapter,
4442                                        struct flash_file_hdr_g3 *fhdr)
4443 {
4444         int ufi_type = be_get_ufi_type(adapter, fhdr);
4445
4446         switch (ufi_type) {
4447         case SH_P2_UFI:
4448                 return skyhawk_chip(adapter);
4449         case SH_UFI:
4450                 return (skyhawk_chip(adapter) &&
4451                         adapter->asic_rev < ASIC_REV_P2);
4452         case BE3R_UFI:
4453                 return BE3_chip(adapter);
4454         case BE3_UFI:
4455                 return (BE3_chip(adapter) && adapter->asic_rev < ASIC_REV_B0);
4456         case BE2_UFI:
4457                 return BE2_chip(adapter);
4458         default:
4459                 return false;
4460         }
4461 }
4462
4463 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4464 {
4465         struct device *dev = &adapter->pdev->dev;
4466         struct flash_file_hdr_g3 *fhdr3;
4467         struct image_hdr *img_hdr_ptr;
4468         int status = 0, i, num_imgs;
4469         struct be_dma_mem flash_cmd;
4470
4471         fhdr3 = (struct flash_file_hdr_g3 *)fw->data;
4472         if (!be_check_ufi_compatibility(adapter, fhdr3)) {
4473                 dev_err(dev, "Flash image is not compatible with adapter\n");
4474                 return -EINVAL;
4475         }
4476
4477         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4478         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size, &flash_cmd.dma,
4479                                           GFP_KERNEL);
4480         if (!flash_cmd.va)
4481                 return -ENOMEM;
4482
4483         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4484         for (i = 0; i < num_imgs; i++) {
4485                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4486                                 (sizeof(struct flash_file_hdr_g3) +
4487                                  i * sizeof(struct image_hdr)));
4488                 if (!BE2_chip(adapter) &&
4489                     le32_to_cpu(img_hdr_ptr->imageid) != 1)
4490                         continue;
4491
4492                 if (skyhawk_chip(adapter))
4493                         status = be_flash_skyhawk(adapter, fw, &flash_cmd,
4494                                                   num_imgs);
4495                 else
4496                         status = be_flash_BEx(adapter, fw, &flash_cmd,
4497                                               num_imgs);
4498         }
4499
4500         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4501         if (!status)
4502                 dev_info(dev, "Firmware flashed successfully\n");
4503
4504         return status;
4505 }
4506
4507 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4508 {
4509         const struct firmware *fw;
4510         int status;
4511
4512         if (!netif_running(adapter->netdev)) {
4513                 dev_err(&adapter->pdev->dev,
4514                         "Firmware load not allowed (interface is down)\n");
4515                 return -ENETDOWN;
4516         }
4517
4518         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4519         if (status)
4520                 goto fw_exit;
4521
4522         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4523
4524         if (lancer_chip(adapter))
4525                 status = lancer_fw_download(adapter, fw);
4526         else
4527                 status = be_fw_download(adapter, fw);
4528
4529         if (!status)
4530                 be_cmd_get_fw_ver(adapter);
4531
4532 fw_exit:
4533         release_firmware(fw);
4534         return status;
4535 }
4536
4537 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
4538                                  u16 flags)
4539 {
4540         struct be_adapter *adapter = netdev_priv(dev);
4541         struct nlattr *attr, *br_spec;
4542         int rem;
4543         int status = 0;
4544         u16 mode = 0;
4545
4546         if (!sriov_enabled(adapter))
4547                 return -EOPNOTSUPP;
4548
4549         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4550         if (!br_spec)
4551                 return -EINVAL;
4552
4553         nla_for_each_nested(attr, br_spec, rem) {
4554                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4555                         continue;
4556
4557                 if (nla_len(attr) < sizeof(mode))
4558                         return -EINVAL;
4559
4560                 mode = nla_get_u16(attr);
4561                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4562                         return -EINVAL;
4563
4564                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4565                                                adapter->if_handle,
4566                                                mode == BRIDGE_MODE_VEPA ?
4567                                                PORT_FWD_TYPE_VEPA :
4568                                                PORT_FWD_TYPE_VEB);
4569                 if (status)
4570                         goto err;
4571
4572                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4573                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4574
4575                 return status;
4576         }
4577 err:
4578         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4579                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4580
4581         return status;
4582 }
4583
4584 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4585                                  struct net_device *dev, u32 filter_mask)
4586 {
4587         struct be_adapter *adapter = netdev_priv(dev);
4588         int status = 0;
4589         u8 hsw_mode;
4590
4591         if (!sriov_enabled(adapter))
4592                 return 0;
4593
4594         /* BE and Lancer chips support VEB mode only */
4595         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4596                 hsw_mode = PORT_FWD_TYPE_VEB;
4597         } else {
4598                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4599                                                adapter->if_handle, &hsw_mode);
4600                 if (status)
4601                         return 0;
4602         }
4603
4604         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4605                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4606                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB,
4607                                        0, 0);
4608 }
4609
4610 #ifdef CONFIG_BE2NET_VXLAN
4611 /* VxLAN offload Notes:
4612  *
4613  * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't
4614  * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload
4615  * is expected to work across all types of IP tunnels once exported. Skyhawk
4616  * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN
4617  * offloads in hw_enc_features only when a VxLAN port is added. If other (non
4618  * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for
4619  * those other tunnels are unexported on the fly through ndo_features_check().
4620  *
4621  * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack
4622  * adds more than one port, disable offloads and don't re-enable them again
4623  * until after all the tunnels are removed.
4624  */
4625 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4626                               __be16 port)
4627 {
4628         struct be_adapter *adapter = netdev_priv(netdev);
4629         struct device *dev = &adapter->pdev->dev;
4630         int status;
4631
4632         if (lancer_chip(adapter) || BEx_chip(adapter))
4633                 return;
4634
4635         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4636                 dev_info(dev,
4637                          "Only one UDP port supported for VxLAN offloads\n");
4638                 dev_info(dev, "Disabling VxLAN offloads\n");
4639                 adapter->vxlan_port_count++;
4640                 goto err;
4641         }
4642
4643         if (adapter->vxlan_port_count++ >= 1)
4644                 return;
4645
4646         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4647                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4648         if (status) {
4649                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4650                 goto err;
4651         }
4652
4653         status = be_cmd_set_vxlan_port(adapter, port);
4654         if (status) {
4655                 dev_warn(dev, "Failed to add VxLAN port\n");
4656                 goto err;
4657         }
4658         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4659         adapter->vxlan_port = port;
4660
4661         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4662                                    NETIF_F_TSO | NETIF_F_TSO6 |
4663                                    NETIF_F_GSO_UDP_TUNNEL;
4664         netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4665         netdev->features |= NETIF_F_GSO_UDP_TUNNEL;
4666
4667         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4668                  be16_to_cpu(port));
4669         return;
4670 err:
4671         be_disable_vxlan_offloads(adapter);
4672 }
4673
4674 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4675                               __be16 port)
4676 {
4677         struct be_adapter *adapter = netdev_priv(netdev);
4678
4679         if (lancer_chip(adapter) || BEx_chip(adapter))
4680                 return;
4681
4682         if (adapter->vxlan_port != port)
4683                 goto done;
4684
4685         be_disable_vxlan_offloads(adapter);
4686
4687         dev_info(&adapter->pdev->dev,
4688                  "Disabled VxLAN offloads for UDP port %d\n",
4689                  be16_to_cpu(port));
4690 done:
4691         adapter->vxlan_port_count--;
4692 }
4693
4694 static netdev_features_t be_features_check(struct sk_buff *skb,
4695                                            struct net_device *dev,
4696                                            netdev_features_t features)
4697 {
4698         struct be_adapter *adapter = netdev_priv(dev);
4699         u8 l4_hdr = 0;
4700
4701         /* The code below restricts offload features for some tunneled packets.
4702          * Offload features for normal (non tunnel) packets are unchanged.
4703          */
4704         if (!skb->encapsulation ||
4705             !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS))
4706                 return features;
4707
4708         /* It's an encapsulated packet and VxLAN offloads are enabled. We
4709          * should disable tunnel offload features if it's not a VxLAN packet,
4710          * as tunnel offloads have been enabled only for VxLAN. This is done to
4711          * allow other tunneled traffic like GRE work fine while VxLAN
4712          * offloads are configured in Skyhawk-R.
4713          */
4714         switch (vlan_get_protocol(skb)) {
4715         case htons(ETH_P_IP):
4716                 l4_hdr = ip_hdr(skb)->protocol;
4717                 break;
4718         case htons(ETH_P_IPV6):
4719                 l4_hdr = ipv6_hdr(skb)->nexthdr;
4720                 break;
4721         default:
4722                 return features;
4723         }
4724
4725         if (l4_hdr != IPPROTO_UDP ||
4726             skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
4727             skb->inner_protocol != htons(ETH_P_TEB) ||
4728             skb_inner_mac_header(skb) - skb_transport_header(skb) !=
4729             sizeof(struct udphdr) + sizeof(struct vxlanhdr))
4730                 return features & ~(NETIF_F_ALL_CSUM | NETIF_F_GSO_MASK);
4731
4732         return features;
4733 }
4734 #endif
4735
4736 static const struct net_device_ops be_netdev_ops = {
4737         .ndo_open               = be_open,
4738         .ndo_stop               = be_close,
4739         .ndo_start_xmit         = be_xmit,
4740         .ndo_set_rx_mode        = be_set_rx_mode,
4741         .ndo_set_mac_address    = be_mac_addr_set,
4742         .ndo_change_mtu         = be_change_mtu,
4743         .ndo_get_stats64        = be_get_stats64,
4744         .ndo_validate_addr      = eth_validate_addr,
4745         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4746         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4747         .ndo_set_vf_mac         = be_set_vf_mac,
4748         .ndo_set_vf_vlan        = be_set_vf_vlan,
4749         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4750         .ndo_get_vf_config      = be_get_vf_config,
4751         .ndo_set_vf_link_state  = be_set_vf_link_state,
4752 #ifdef CONFIG_NET_POLL_CONTROLLER
4753         .ndo_poll_controller    = be_netpoll,
4754 #endif
4755         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4756         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4757 #ifdef CONFIG_NET_RX_BUSY_POLL
4758         .ndo_busy_poll          = be_busy_poll,
4759 #endif
4760 #ifdef CONFIG_BE2NET_VXLAN
4761         .ndo_add_vxlan_port     = be_add_vxlan_port,
4762         .ndo_del_vxlan_port     = be_del_vxlan_port,
4763         .ndo_features_check     = be_features_check,
4764 #endif
4765 };
4766
4767 static void be_netdev_init(struct net_device *netdev)
4768 {
4769         struct be_adapter *adapter = netdev_priv(netdev);
4770
4771         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4772                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4773                 NETIF_F_HW_VLAN_CTAG_TX;
4774         if (be_multi_rxq(adapter))
4775                 netdev->hw_features |= NETIF_F_RXHASH;
4776
4777         netdev->features |= netdev->hw_features |
4778                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4779
4780         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4781                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4782
4783         netdev->priv_flags |= IFF_UNICAST_FLT;
4784
4785         netdev->flags |= IFF_MULTICAST;
4786
4787         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4788
4789         netdev->netdev_ops = &be_netdev_ops;
4790
4791         netdev->ethtool_ops = &be_ethtool_ops;
4792 }
4793
4794 static void be_unmap_pci_bars(struct be_adapter *adapter)
4795 {
4796         if (adapter->csr)
4797                 pci_iounmap(adapter->pdev, adapter->csr);
4798         if (adapter->db)
4799                 pci_iounmap(adapter->pdev, adapter->db);
4800 }
4801
4802 static int db_bar(struct be_adapter *adapter)
4803 {
4804         if (lancer_chip(adapter) || !be_physfn(adapter))
4805                 return 0;
4806         else
4807                 return 4;
4808 }
4809
4810 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4811 {
4812         if (skyhawk_chip(adapter)) {
4813                 adapter->roce_db.size = 4096;
4814                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4815                                                               db_bar(adapter));
4816                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4817                                                                db_bar(adapter));
4818         }
4819         return 0;
4820 }
4821
4822 static int be_map_pci_bars(struct be_adapter *adapter)
4823 {
4824         u8 __iomem *addr;
4825
4826         if (BEx_chip(adapter) && be_physfn(adapter)) {
4827                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4828                 if (!adapter->csr)
4829                         return -ENOMEM;
4830         }
4831
4832         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4833         if (!addr)
4834                 goto pci_map_err;
4835         adapter->db = addr;
4836
4837         be_roce_map_pci_bars(adapter);
4838         return 0;
4839
4840 pci_map_err:
4841         dev_err(&adapter->pdev->dev, "Error in mapping PCI BARs\n");
4842         be_unmap_pci_bars(adapter);
4843         return -ENOMEM;
4844 }
4845
4846 static void be_ctrl_cleanup(struct be_adapter *adapter)
4847 {
4848         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4849
4850         be_unmap_pci_bars(adapter);
4851
4852         if (mem->va)
4853                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4854                                   mem->dma);
4855
4856         mem = &adapter->rx_filter;
4857         if (mem->va)
4858                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4859                                   mem->dma);
4860 }
4861
4862 static int be_ctrl_init(struct be_adapter *adapter)
4863 {
4864         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4865         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4866         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4867         u32 sli_intf;
4868         int status;
4869
4870         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4871         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4872                                  SLI_INTF_FAMILY_SHIFT;
4873         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4874
4875         status = be_map_pci_bars(adapter);
4876         if (status)
4877                 goto done;
4878
4879         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4880         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4881                                                 mbox_mem_alloc->size,
4882                                                 &mbox_mem_alloc->dma,
4883                                                 GFP_KERNEL);
4884         if (!mbox_mem_alloc->va) {
4885                 status = -ENOMEM;
4886                 goto unmap_pci_bars;
4887         }
4888         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4889         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4890         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4891         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4892
4893         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4894         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4895                                             rx_filter->size, &rx_filter->dma,
4896                                             GFP_KERNEL);
4897         if (!rx_filter->va) {
4898                 status = -ENOMEM;
4899                 goto free_mbox;
4900         }
4901
4902         mutex_init(&adapter->mbox_lock);
4903         spin_lock_init(&adapter->mcc_lock);
4904         spin_lock_init(&adapter->mcc_cq_lock);
4905
4906         init_completion(&adapter->et_cmd_compl);
4907         pci_save_state(adapter->pdev);
4908         return 0;
4909
4910 free_mbox:
4911         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4912                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4913
4914 unmap_pci_bars:
4915         be_unmap_pci_bars(adapter);
4916
4917 done:
4918         return status;
4919 }
4920
4921 static void be_stats_cleanup(struct be_adapter *adapter)
4922 {
4923         struct be_dma_mem *cmd = &adapter->stats_cmd;
4924
4925         if (cmd->va)
4926                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4927                                   cmd->va, cmd->dma);
4928 }
4929
4930 static int be_stats_init(struct be_adapter *adapter)
4931 {
4932         struct be_dma_mem *cmd = &adapter->stats_cmd;
4933
4934         if (lancer_chip(adapter))
4935                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
4936         else if (BE2_chip(adapter))
4937                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
4938         else if (BE3_chip(adapter))
4939                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
4940         else
4941                 /* ALL non-BE ASICs */
4942                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
4943
4944         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
4945                                       GFP_KERNEL);
4946         if (!cmd->va)
4947                 return -ENOMEM;
4948         return 0;
4949 }
4950
4951 static void be_remove(struct pci_dev *pdev)
4952 {
4953         struct be_adapter *adapter = pci_get_drvdata(pdev);
4954
4955         if (!adapter)
4956                 return;
4957
4958         be_roce_dev_remove(adapter);
4959         be_intr_set(adapter, false);
4960
4961         cancel_delayed_work_sync(&adapter->func_recovery_work);
4962
4963         unregister_netdev(adapter->netdev);
4964
4965         be_clear(adapter);
4966
4967         /* tell fw we're done with firing cmds */
4968         be_cmd_fw_clean(adapter);
4969
4970         be_stats_cleanup(adapter);
4971
4972         be_ctrl_cleanup(adapter);
4973
4974         pci_disable_pcie_error_reporting(pdev);
4975
4976         pci_release_regions(pdev);
4977         pci_disable_device(pdev);
4978
4979         free_netdev(adapter->netdev);
4980 }
4981
4982 static int be_get_initial_config(struct be_adapter *adapter)
4983 {
4984         int status, level;
4985
4986         status = be_cmd_get_cntl_attributes(adapter);
4987         if (status)
4988                 return status;
4989
4990         /* Must be a power of 2 or else MODULO will BUG_ON */
4991         adapter->be_get_temp_freq = 64;
4992
4993         if (BEx_chip(adapter)) {
4994                 level = be_cmd_get_fw_log_level(adapter);
4995                 adapter->msg_enable =
4996                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4997         }
4998
4999         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
5000         return 0;
5001 }
5002
5003 static int lancer_recover_func(struct be_adapter *adapter)
5004 {
5005         struct device *dev = &adapter->pdev->dev;
5006         int status;
5007
5008         status = lancer_test_and_set_rdy_state(adapter);
5009         if (status)
5010                 goto err;
5011
5012         if (netif_running(adapter->netdev))
5013                 be_close(adapter->netdev);
5014
5015         be_clear(adapter);
5016
5017         be_clear_all_error(adapter);
5018
5019         status = be_setup(adapter);
5020         if (status)
5021                 goto err;
5022
5023         if (netif_running(adapter->netdev)) {
5024                 status = be_open(adapter->netdev);
5025                 if (status)
5026                         goto err;
5027         }
5028
5029         dev_err(dev, "Adapter recovery successful\n");
5030         return 0;
5031 err:
5032         if (status == -EAGAIN)
5033                 dev_err(dev, "Waiting for resource provisioning\n");
5034         else
5035                 dev_err(dev, "Adapter recovery failed\n");
5036
5037         return status;
5038 }
5039
5040 static void be_func_recovery_task(struct work_struct *work)
5041 {
5042         struct be_adapter *adapter =
5043                 container_of(work, struct be_adapter,  func_recovery_work.work);
5044         int status = 0;
5045
5046         be_detect_error(adapter);
5047
5048         if (adapter->hw_error && lancer_chip(adapter)) {
5049                 rtnl_lock();
5050                 netif_device_detach(adapter->netdev);
5051                 rtnl_unlock();
5052
5053                 status = lancer_recover_func(adapter);
5054                 if (!status)
5055                         netif_device_attach(adapter->netdev);
5056         }
5057
5058         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
5059          * no need to attempt further recovery.
5060          */
5061         if (!status || status == -EAGAIN)
5062                 schedule_delayed_work(&adapter->func_recovery_work,
5063                                       msecs_to_jiffies(1000));
5064 }
5065
5066 static void be_log_sfp_info(struct be_adapter *adapter)
5067 {
5068         int status;
5069
5070         status = be_cmd_query_sfp_info(adapter);
5071         if (!status) {
5072                 dev_err(&adapter->pdev->dev,
5073                         "Unqualified SFP+ detected on %c from %s part no: %s",
5074                         adapter->port_name, adapter->phy.vendor_name,
5075                         adapter->phy.vendor_pn);
5076         }
5077         adapter->flags &= ~BE_FLAGS_EVT_INCOMPATIBLE_SFP;
5078 }
5079
5080 static void be_worker(struct work_struct *work)
5081 {
5082         struct be_adapter *adapter =
5083                 container_of(work, struct be_adapter, work.work);
5084         struct be_rx_obj *rxo;
5085         int i;
5086
5087         /* when interrupts are not yet enabled, just reap any pending
5088         * mcc completions */
5089         if (!netif_running(adapter->netdev)) {
5090                 local_bh_disable();
5091                 be_process_mcc(adapter);
5092                 local_bh_enable();
5093                 goto reschedule;
5094         }
5095
5096         if (!adapter->stats_cmd_sent) {
5097                 if (lancer_chip(adapter))
5098                         lancer_cmd_get_pport_stats(adapter,
5099                                                    &adapter->stats_cmd);
5100                 else
5101                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
5102         }
5103
5104         if (be_physfn(adapter) &&
5105             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
5106                 be_cmd_get_die_temperature(adapter);
5107
5108         for_all_rx_queues(adapter, rxo, i) {
5109                 /* Replenish RX-queues starved due to memory
5110                  * allocation failures.
5111                  */
5112                 if (rxo->rx_post_starved)
5113                         be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
5114         }
5115
5116         be_eqd_update(adapter);
5117
5118         if (adapter->flags & BE_FLAGS_EVT_INCOMPATIBLE_SFP)
5119                 be_log_sfp_info(adapter);
5120
5121 reschedule:
5122         adapter->work_counter++;
5123         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
5124 }
5125
5126 /* If any VFs are already enabled don't FLR the PF */
5127 static bool be_reset_required(struct be_adapter *adapter)
5128 {
5129         return pci_num_vf(adapter->pdev) ? false : true;
5130 }
5131
5132 static char *mc_name(struct be_adapter *adapter)
5133 {
5134         char *str = ""; /* default */
5135
5136         switch (adapter->mc_type) {
5137         case UMC:
5138                 str = "UMC";
5139                 break;
5140         case FLEX10:
5141                 str = "FLEX10";
5142                 break;
5143         case vNIC1:
5144                 str = "vNIC-1";
5145                 break;
5146         case nPAR:
5147                 str = "nPAR";
5148                 break;
5149         case UFP:
5150                 str = "UFP";
5151                 break;
5152         case vNIC2:
5153                 str = "vNIC-2";
5154                 break;
5155         default:
5156                 str = "";
5157         }
5158
5159         return str;
5160 }
5161
5162 static inline char *func_name(struct be_adapter *adapter)
5163 {
5164         return be_physfn(adapter) ? "PF" : "VF";
5165 }
5166
5167 static inline char *nic_name(struct pci_dev *pdev)
5168 {
5169         switch (pdev->device) {
5170         case OC_DEVICE_ID1:
5171                 return OC_NAME;
5172         case OC_DEVICE_ID2:
5173                 return OC_NAME_BE;
5174         case OC_DEVICE_ID3:
5175         case OC_DEVICE_ID4:
5176                 return OC_NAME_LANCER;
5177         case BE_DEVICE_ID2:
5178                 return BE3_NAME;
5179         case OC_DEVICE_ID5:
5180         case OC_DEVICE_ID6:
5181                 return OC_NAME_SH;
5182         default:
5183                 return BE_NAME;
5184         }
5185 }
5186
5187 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
5188 {
5189         struct be_adapter *adapter;
5190         struct net_device *netdev;
5191         int status = 0;
5192
5193         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
5194
5195         status = pci_enable_device(pdev);
5196         if (status)
5197                 goto do_none;
5198
5199         status = pci_request_regions(pdev, DRV_NAME);
5200         if (status)
5201                 goto disable_dev;
5202         pci_set_master(pdev);
5203
5204         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
5205         if (!netdev) {
5206                 status = -ENOMEM;
5207                 goto rel_reg;
5208         }
5209         adapter = netdev_priv(netdev);
5210         adapter->pdev = pdev;
5211         pci_set_drvdata(pdev, adapter);
5212         adapter->netdev = netdev;
5213         SET_NETDEV_DEV(netdev, &pdev->dev);
5214
5215         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
5216         if (!status) {
5217                 netdev->features |= NETIF_F_HIGHDMA;
5218         } else {
5219                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
5220                 if (status) {
5221                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
5222                         goto free_netdev;
5223                 }
5224         }
5225
5226         status = pci_enable_pcie_error_reporting(pdev);
5227         if (!status)
5228                 dev_info(&pdev->dev, "PCIe error reporting enabled\n");
5229
5230         status = be_ctrl_init(adapter);
5231         if (status)
5232                 goto free_netdev;
5233
5234         /* sync up with fw's ready state */
5235         if (be_physfn(adapter)) {
5236                 status = be_fw_wait_ready(adapter);
5237                 if (status)
5238                         goto ctrl_clean;
5239         }
5240
5241         if (be_reset_required(adapter)) {
5242                 status = be_cmd_reset_function(adapter);
5243                 if (status)
5244                         goto ctrl_clean;
5245
5246                 /* Wait for interrupts to quiesce after an FLR */
5247                 msleep(100);
5248         }
5249
5250         /* Allow interrupts for other ULPs running on NIC function */
5251         be_intr_set(adapter, true);
5252
5253         /* tell fw we're ready to fire cmds */
5254         status = be_cmd_fw_init(adapter);
5255         if (status)
5256                 goto ctrl_clean;
5257
5258         status = be_stats_init(adapter);
5259         if (status)
5260                 goto ctrl_clean;
5261
5262         status = be_get_initial_config(adapter);
5263         if (status)
5264                 goto stats_clean;
5265
5266         INIT_DELAYED_WORK(&adapter->work, be_worker);
5267         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
5268         adapter->rx_fc = true;
5269         adapter->tx_fc = true;
5270
5271         status = be_setup(adapter);
5272         if (status)
5273                 goto stats_clean;
5274
5275         be_netdev_init(netdev);
5276         status = register_netdev(netdev);
5277         if (status != 0)
5278                 goto unsetup;
5279
5280         be_roce_dev_add(adapter);
5281
5282         schedule_delayed_work(&adapter->func_recovery_work,
5283                               msecs_to_jiffies(1000));
5284
5285         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
5286                  func_name(adapter), mc_name(adapter), adapter->port_name);
5287
5288         return 0;
5289
5290 unsetup:
5291         be_clear(adapter);
5292 stats_clean:
5293         be_stats_cleanup(adapter);
5294 ctrl_clean:
5295         be_ctrl_cleanup(adapter);
5296 free_netdev:
5297         free_netdev(netdev);
5298 rel_reg:
5299         pci_release_regions(pdev);
5300 disable_dev:
5301         pci_disable_device(pdev);
5302 do_none:
5303         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
5304         return status;
5305 }
5306
5307 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
5308 {
5309         struct be_adapter *adapter = pci_get_drvdata(pdev);
5310         struct net_device *netdev =  adapter->netdev;
5311
5312         if (adapter->wol_en)
5313                 be_setup_wol(adapter, true);
5314
5315         be_intr_set(adapter, false);
5316         cancel_delayed_work_sync(&adapter->func_recovery_work);
5317
5318         netif_device_detach(netdev);
5319         if (netif_running(netdev)) {
5320                 rtnl_lock();
5321                 be_close(netdev);
5322                 rtnl_unlock();
5323         }
5324         be_clear(adapter);
5325
5326         pci_save_state(pdev);
5327         pci_disable_device(pdev);
5328         pci_set_power_state(pdev, pci_choose_state(pdev, state));
5329         return 0;
5330 }
5331
5332 static int be_resume(struct pci_dev *pdev)
5333 {
5334         int status = 0;
5335         struct be_adapter *adapter = pci_get_drvdata(pdev);
5336         struct net_device *netdev =  adapter->netdev;
5337
5338         netif_device_detach(netdev);
5339
5340         status = pci_enable_device(pdev);
5341         if (status)
5342                 return status;
5343
5344         pci_set_power_state(pdev, PCI_D0);
5345         pci_restore_state(pdev);
5346
5347         status = be_fw_wait_ready(adapter);
5348         if (status)
5349                 return status;
5350
5351         status = be_cmd_reset_function(adapter);
5352         if (status)
5353                 return status;
5354
5355         be_intr_set(adapter, true);
5356         /* tell fw we're ready to fire cmds */
5357         status = be_cmd_fw_init(adapter);
5358         if (status)
5359                 return status;
5360
5361         be_setup(adapter);
5362         if (netif_running(netdev)) {
5363                 rtnl_lock();
5364                 be_open(netdev);
5365                 rtnl_unlock();
5366         }
5367
5368         schedule_delayed_work(&adapter->func_recovery_work,
5369                               msecs_to_jiffies(1000));
5370         netif_device_attach(netdev);
5371
5372         if (adapter->wol_en)
5373                 be_setup_wol(adapter, false);
5374
5375         return 0;
5376 }
5377
5378 /*
5379  * An FLR will stop BE from DMAing any data.
5380  */
5381 static void be_shutdown(struct pci_dev *pdev)
5382 {
5383         struct be_adapter *adapter = pci_get_drvdata(pdev);
5384
5385         if (!adapter)
5386                 return;
5387
5388         be_roce_dev_shutdown(adapter);
5389         cancel_delayed_work_sync(&adapter->work);
5390         cancel_delayed_work_sync(&adapter->func_recovery_work);
5391
5392         netif_device_detach(adapter->netdev);
5393
5394         be_cmd_reset_function(adapter);
5395
5396         pci_disable_device(pdev);
5397 }
5398
5399 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5400                                             pci_channel_state_t state)
5401 {
5402         struct be_adapter *adapter = pci_get_drvdata(pdev);
5403         struct net_device *netdev =  adapter->netdev;
5404
5405         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5406
5407         if (!adapter->eeh_error) {
5408                 adapter->eeh_error = true;
5409
5410                 cancel_delayed_work_sync(&adapter->func_recovery_work);
5411
5412                 rtnl_lock();
5413                 netif_device_detach(netdev);
5414                 if (netif_running(netdev))
5415                         be_close(netdev);
5416                 rtnl_unlock();
5417
5418                 be_clear(adapter);
5419         }
5420
5421         if (state == pci_channel_io_perm_failure)
5422                 return PCI_ERS_RESULT_DISCONNECT;
5423
5424         pci_disable_device(pdev);
5425
5426         /* The error could cause the FW to trigger a flash debug dump.
5427          * Resetting the card while flash dump is in progress
5428          * can cause it not to recover; wait for it to finish.
5429          * Wait only for first function as it is needed only once per
5430          * adapter.
5431          */
5432         if (pdev->devfn == 0)
5433                 ssleep(30);
5434
5435         return PCI_ERS_RESULT_NEED_RESET;
5436 }
5437
5438 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5439 {
5440         struct be_adapter *adapter = pci_get_drvdata(pdev);
5441         int status;
5442
5443         dev_info(&adapter->pdev->dev, "EEH reset\n");
5444
5445         status = pci_enable_device(pdev);
5446         if (status)
5447                 return PCI_ERS_RESULT_DISCONNECT;
5448
5449         pci_set_master(pdev);
5450         pci_set_power_state(pdev, PCI_D0);
5451         pci_restore_state(pdev);
5452
5453         /* Check if card is ok and fw is ready */
5454         dev_info(&adapter->pdev->dev,
5455                  "Waiting for FW to be ready after EEH reset\n");
5456         status = be_fw_wait_ready(adapter);
5457         if (status)
5458                 return PCI_ERS_RESULT_DISCONNECT;
5459
5460         pci_cleanup_aer_uncorrect_error_status(pdev);
5461         be_clear_all_error(adapter);
5462         return PCI_ERS_RESULT_RECOVERED;
5463 }
5464
5465 static void be_eeh_resume(struct pci_dev *pdev)
5466 {
5467         int status = 0;
5468         struct be_adapter *adapter = pci_get_drvdata(pdev);
5469         struct net_device *netdev =  adapter->netdev;
5470
5471         dev_info(&adapter->pdev->dev, "EEH resume\n");
5472
5473         pci_save_state(pdev);
5474
5475         status = be_cmd_reset_function(adapter);
5476         if (status)
5477                 goto err;
5478
5479         /* On some BE3 FW versions, after a HW reset,
5480          * interrupts will remain disabled for each function.
5481          * So, explicitly enable interrupts
5482          */
5483         be_intr_set(adapter, true);
5484
5485         /* tell fw we're ready to fire cmds */
5486         status = be_cmd_fw_init(adapter);
5487         if (status)
5488                 goto err;
5489
5490         status = be_setup(adapter);
5491         if (status)
5492                 goto err;
5493
5494         if (netif_running(netdev)) {
5495                 status = be_open(netdev);
5496                 if (status)
5497                         goto err;
5498         }
5499
5500         schedule_delayed_work(&adapter->func_recovery_work,
5501                               msecs_to_jiffies(1000));
5502         netif_device_attach(netdev);
5503         return;
5504 err:
5505         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5506 }
5507
5508 static const struct pci_error_handlers be_eeh_handlers = {
5509         .error_detected = be_eeh_err_detected,
5510         .slot_reset = be_eeh_reset,
5511         .resume = be_eeh_resume,
5512 };
5513
5514 static struct pci_driver be_driver = {
5515         .name = DRV_NAME,
5516         .id_table = be_dev_ids,
5517         .probe = be_probe,
5518         .remove = be_remove,
5519         .suspend = be_suspend,
5520         .resume = be_resume,
5521         .shutdown = be_shutdown,
5522         .err_handler = &be_eeh_handlers
5523 };
5524
5525 static int __init be_init_module(void)
5526 {
5527         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5528             rx_frag_size != 2048) {
5529                 printk(KERN_WARNING DRV_NAME
5530                         " : Module param rx_frag_size must be 2048/4096/8192."
5531                         " Using 2048\n");
5532                 rx_frag_size = 2048;
5533         }
5534
5535         return pci_register_driver(&be_driver);
5536 }
5537 module_init(be_init_module);
5538
5539 static void __exit be_exit_module(void)
5540 {
5541         pci_unregister_driver(&be_driver);
5542 }
5543 module_exit(be_exit_module);