drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2013 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/tcp.h>
  25 #include <net/ipv6.h>
  26 #include <net/ip6_checksum.h>
  27 #include <linux/prefetch.h>
  28 #include "bnx2x_cmn.h"
  29 #include "bnx2x_init.h"
  30 #include "bnx2x_sp.h"
  31
  32 /**
  33  * bnx2x_move_fp - move content of the fastpath structure.
  34  *
  35  * @bp:         driver handle
  36  * @from:       source FP index
  37  * @to:         destination FP index
  38  *
  39  * Makes sure the contents of the bp->fp[to].napi is kept
  40  * intact. This is done by first copying the napi struct from
  41  * the target to the source, and then mem copying the entire
  42  * source onto the target. Update txdata pointers and related
  43  * content.
  44  */
  45 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  46 {
  47         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  48         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  49         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  50         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  51         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  52         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  53         int old_max_eth_txqs, new_max_eth_txqs;
  54         int old_txdata_index = 0, new_txdata_index = 0;
  55
  56         /* Copy the NAPI object as it has been already initialized */
  57         from_fp->napi = to_fp->napi;
  58
  59         /* Move bnx2x_fastpath contents */
  60         memcpy(to_fp, from_fp, sizeof(*to_fp));
  61         to_fp->index = to;
  62
  63         /* move sp_objs contents as well, as their indices match fp ones */
  64         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
  65
  66         /* move fp_stats contents as well, as their indices match fp ones */
  67         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
  68
  69         /* Update txdata pointers in fp and move txdata content accordingly:
  70          * Each fp consumes 'max_cos' txdata structures, so the index should be
  71          * decremented by max_cos x delta.
  72          */
  73
  74         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
  75         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
  76                                 (bp)->max_cos;
  77         if (from == FCOE_IDX(bp)) {
  78                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  79                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  80         }
  81
  82         memcpy(&bp->bnx2x_txq[new_txdata_index],
  83                &bp->bnx2x_txq[old_txdata_index],
  84                sizeof(struct bnx2x_fp_txdata));
  85         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
  86 }
  87
  88 /**
  89  * bnx2x_fill_fw_str - Fill buffer with FW version string.
  90  *
  91  * @bp:        driver handle
  92  * @buf:       character buffer to fill with the fw name
  93  * @buf_len:   length of the above buffer
  94  *
  95  */
  96 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
  97 {
  98         if (IS_PF(bp)) {
  99                 u8 phy_fw_ver[PHY_FW_VER_LEN];
 100
 101                 phy_fw_ver[0] = '\0';
 102                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
 103                                              phy_fw_ver, PHY_FW_VER_LEN);
 104                 strlcpy(buf, bp->fw_ver, buf_len);
 105                 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
 106                          "bc %d.%d.%d%s%s",
 107                          (bp->common.bc_ver & 0xff0000) >> 16,
 108                          (bp->common.bc_ver & 0xff00) >> 8,
 109                          (bp->common.bc_ver & 0xff),
 110                          ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
 111         } else {
 112                 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
 113         }
 114 }
 115
 116 /**
 117  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
 118  *
 119  * @bp: driver handle
 120  * @delta:      number of eth queues which were not allocated
 121  */
 122 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
 123 {
 124         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
 125
 126         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
 127          * backward along the array could cause memory to be overriden
 128          */
 129         for (cos = 1; cos < bp->max_cos; cos++) {
 130                 for (i = 0; i < old_eth_num - delta; i++) {
 131                         struct bnx2x_fastpath *fp = &bp->fp[i];
 132                         int new_idx = cos * (old_eth_num - delta) + i;
 133
 134                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
 135                                sizeof(struct bnx2x_fp_txdata));
 136                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
 137                 }
 138         }
 139 }
 140
 141 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
 142
 143 /* free skb in the packet ring at pos idx
 144  * return idx of last bd freed
 145  */
 146 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
 147                              u16 idx, unsigned int *pkts_compl,
 148                              unsigned int *bytes_compl)
 149 {
 150         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
 151         struct eth_tx_start_bd *tx_start_bd;
 152         struct eth_tx_bd *tx_data_bd;
 153         struct sk_buff *skb = tx_buf->skb;
 154         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 155         int nbd;
 156
 157         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 158         prefetch(&skb->end);
 159
 160         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 161            txdata->txq_index, idx, tx_buf, skb);
 162
 163         /* unmap first bd */
 164         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 165         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 166                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
 167
 168
 169         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 170 #ifdef BNX2X_STOP_ON_ERROR
 171         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 172                 BNX2X_ERR("BAD nbd!\n");
 173                 bnx2x_panic();
 174         }
 175 #endif
 176         new_cons = nbd + tx_buf->first_bd;
 177
 178         /* Get the next bd */
 179         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 180
 181         /* Skip a parse bd... */
 182         --nbd;
 183         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 184
 185         /* ...and the TSO split header bd since they have no mapping */
 186         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 187                 --nbd;
 188                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 189         }
 190
 191         /* now free frags */
 192         while (nbd > 0) {
 193
 194                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 195                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 196                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 197                 if (--nbd)
 198                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 199         }
 200
 201         /* release skb */
 202         WARN_ON(!skb);
 203         if (likely(skb)) {
 204                 (*pkts_compl)++;
 205                 (*bytes_compl) += skb->len;
 206         }
 207
 208         dev_kfree_skb_any(skb);
 209         tx_buf->first_bd = 0;
 210         tx_buf->skb = NULL;
 211
 212         return new_cons;
 213 }
 214
 215 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 216 {
 217         struct netdev_queue *txq;
 218         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 219         unsigned int pkts_compl = 0, bytes_compl = 0;
 220
 221 #ifdef BNX2X_STOP_ON_ERROR
 222         if (unlikely(bp->panic))
 223                 return -1;
 224 #endif
 225
 226         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 227         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 228         sw_cons = txdata->tx_pkt_cons;
 229
 230         while (sw_cons != hw_cons) {
 231                 u16 pkt_cons;
 232
 233                 pkt_cons = TX_BD(sw_cons);
 234
 235                 DP(NETIF_MSG_TX_DONE,
 236                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 237                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 238
 239                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 240                                             &pkts_compl, &bytes_compl);
 241
 242                 sw_cons++;
 243         }
 244
 245         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 246
 247         txdata->tx_pkt_cons = sw_cons;
 248         txdata->tx_bd_cons = bd_cons;
 249
 250         /* Need to make the tx_bd_cons update visible to start_xmit()
 251          * before checking for netif_tx_queue_stopped().  Without the
 252          * memory barrier, there is a small possibility that
 253          * start_xmit() will miss it and cause the queue to be stopped
 254          * forever.
 255          * On the other hand we need an rmb() here to ensure the proper
 256          * ordering of bit testing in the following
 257          * netif_tx_queue_stopped(txq) call.
 258          */
 259         smp_mb();
 260
 261         if (unlikely(netif_tx_queue_stopped(txq))) {
 262                 /* Taking tx_lock() is needed to prevent reenabling the queue
 263                  * while it's empty. This could have happen if rx_action() gets
 264                  * suspended in bnx2x_tx_int() after the condition before
 265                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 266                  *
 267                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 268                  * sends some packets consuming the whole queue again->
 269                  * stops the queue
 270                  */
 271
 272                 __netif_tx_lock(txq, smp_processor_id());
 273
 274                 if ((netif_tx_queue_stopped(txq)) &&
 275                     (bp->state == BNX2X_STATE_OPEN) &&
 276                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 277                         netif_tx_wake_queue(txq);
 278
 279                 __netif_tx_unlock(txq);
 280         }
 281         return 0;
 282 }
 283
 284 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 285                                              u16 idx)
 286 {
 287         u16 last_max = fp->last_max_sge;
 288
 289         if (SUB_S16(idx, last_max) > 0)
 290                 fp->last_max_sge = idx;
 291 }
 292
 293 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 294                                          u16 sge_len,
 295                                          struct eth_end_agg_rx_cqe *cqe)
 296 {
 297         struct bnx2x *bp = fp->bp;
 298         u16 last_max, last_elem, first_elem;
 299         u16 delta = 0;
 300         u16 i;
 301
 302         if (!sge_len)
 303                 return;
 304
 305         /* First mark all used pages */
 306         for (i = 0; i < sge_len; i++)
 307                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 308                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 309
 310         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 311            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 312
 313         /* Here we assume that the last SGE index is the biggest */
 314         prefetch((void *)(fp->sge_mask));
 315         bnx2x_update_last_max_sge(fp,
 316                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 317
 318         last_max = RX_SGE(fp->last_max_sge);
 319         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 320         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 321
 322         /* If ring is not full */
 323         if (last_elem + 1 != first_elem)
 324                 last_elem++;
 325
 326         /* Now update the prod */
 327         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 328                 if (likely(fp->sge_mask[i]))
 329                         break;
 330
 331                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 332                 delta += BIT_VEC64_ELEM_SZ;
 333         }
 334
 335         if (delta > 0) {
 336                 fp->rx_sge_prod += delta;
 337                 /* clear page-end entries */
 338                 bnx2x_clear_sge_mask_next_elems(fp);
 339         }
 340
 341         DP(NETIF_MSG_RX_STATUS,
 342            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 343            fp->last_max_sge, fp->rx_sge_prod);
 344 }
 345
 346 /* Get Toeplitz hash value in the skb using the value from the
 347  * CQE (calculated by HW).
 348  */
 349 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 350                             const struct eth_fast_path_rx_cqe *cqe,
 351                             bool *l4_rxhash)
 352 {
 353         /* Get Toeplitz hash from CQE */
 354         if ((bp->dev->features & NETIF_F_RXHASH) &&
 355             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 356                 enum eth_rss_hash_type htype;
 357
 358                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 359                 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
 360                              (htype == TCP_IPV6_HASH_TYPE);
 361                 return le32_to_cpu(cqe->rss_hash_result);
 362         }
 363         *l4_rxhash = false;
 364         return 0;
 365 }
 366
 367 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 368                             u16 cons, u16 prod,
 369                             struct eth_fast_path_rx_cqe *cqe)
 370 {
 371         struct bnx2x *bp = fp->bp;
 372         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 373         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 374         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 375         dma_addr_t mapping;
 376         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 377         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 378
 379         /* print error if current state != stop */
 380         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 381                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 382
 383         /* Try to map an empty data buffer from the aggregation info  */
 384         mapping = dma_map_single(&bp->pdev->dev,
 385                                  first_buf->data + NET_SKB_PAD,
 386                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 387         /*
 388          *  ...if it fails - move the skb from the consumer to the producer
 389          *  and set the current aggregation state as ERROR to drop it
 390          *  when TPA_STOP arrives.
 391          */
 392
 393         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 394                 /* Move the BD from the consumer to the producer */
 395                 bnx2x_reuse_rx_data(fp, cons, prod);
 396                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 397                 return;
 398         }
 399
 400         /* move empty data from pool to prod */
 401         prod_rx_buf->data = first_buf->data;
 402         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 403         /* point prod_bd to new data */
 404         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 405         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 406
 407         /* move partial skb from cons to pool (don't unmap yet) */
 408         *first_buf = *cons_rx_buf;
 409
 410         /* mark bin state as START */
 411         tpa_info->parsing_flags =
 412                 le16_to_cpu(cqe->pars_flags.flags);
 413         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 414         tpa_info->tpa_state = BNX2X_TPA_START;
 415         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 416         tpa_info->placement_offset = cqe->placement_offset;
 417         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
 418         if (fp->mode == TPA_MODE_GRO) {
 419                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 420                 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
 421                 tpa_info->gro_size = gro_size;
 422         }
 423
 424 #ifdef BNX2X_STOP_ON_ERROR
 425         fp->tpa_queue_used |= (1 << queue);
 426 #ifdef _ASM_GENERIC_INT_L64_H
 427         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 428 #else
 429         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 430 #endif
 431            fp->tpa_queue_used);
 432 #endif
 433 }
 434
 435 /* Timestamp option length allowed for TPA aggregation:
 436  *
 437  *              nop nop kind length echo val
 438  */
 439 #define TPA_TSTAMP_OPT_LEN      12
 440 /**
 441  * bnx2x_set_gro_params - compute GRO values
 442  *
 443  * @skb:                packet skb
 444  * @parsing_flags:      parsing flags from the START CQE
 445  * @len_on_bd:          total length of the first packet for the
 446  *                      aggregation.
 447  * @pkt_len:            length of all segments
 448  *
 449  * Approximate value of the MSS for this aggregation calculated using
 450  * the first packet of it.
 451  * Compute number of aggregated segments, and gso_type.
 452  */
 453 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
 454                                  u16 len_on_bd, unsigned int pkt_len,
 455                                  u16 num_of_coalesced_segs)
 456 {
 457         /* TPA aggregation won't have either IP options or TCP options
 458          * other than timestamp or IPv6 extension headers.
 459          */
 460         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 461
 462         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 463             PRS_FLAG_OVERETH_IPV6) {
 464                 hdrs_len += sizeof(struct ipv6hdr);
 465                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
 466         } else {
 467                 hdrs_len += sizeof(struct iphdr);
 468                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 469         }
 470
 471         /* Check if there was a TCP timestamp, if there is it's will
 472          * always be 12 bytes length: nop nop kind length echo val.
 473          *
 474          * Otherwise FW would close the aggregation.
 475          */
 476         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 477                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 478
 479         skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
 480
 481         /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
 482          * to skb_shinfo(skb)->gso_segs
 483          */
 484         NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
 485 }
 486
 487 static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
 488                               struct bnx2x_fastpath *fp, u16 index)
 489 {
 490         struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
 491         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 492         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 493         dma_addr_t mapping;
 494
 495         if (unlikely(page == NULL)) {
 496                 BNX2X_ERR("Can't alloc sge\n");
 497                 return -ENOMEM;
 498         }
 499
 500         mapping = dma_map_page(&bp->pdev->dev, page, 0,
 501                                SGE_PAGES, DMA_FROM_DEVICE);
 502         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 503                 __free_pages(page, PAGES_PER_SGE_SHIFT);
 504                 BNX2X_ERR("Can't map sge\n");
 505                 return -ENOMEM;
 506         }
 507
 508         sw_buf->page = page;
 509         dma_unmap_addr_set(sw_buf, mapping, mapping);
 510
 511         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 512         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 513
 514         return 0;
 515 }
 516
 517 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 518                                struct bnx2x_agg_info *tpa_info,
 519                                u16 pages,
 520                                struct sk_buff *skb,
 521                                struct eth_end_agg_rx_cqe *cqe,
 522                                u16 cqe_idx)
 523 {
 524         struct sw_rx_page *rx_pg, old_rx_pg;
 525         u32 i, frag_len, frag_size;
 526         int err, j, frag_id = 0;
 527         u16 len_on_bd = tpa_info->len_on_bd;
 528         u16 full_page = 0, gro_size = 0;
 529
 530         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 531
 532         if (fp->mode == TPA_MODE_GRO) {
 533                 gro_size = tpa_info->gro_size;
 534                 full_page = tpa_info->full_page;
 535         }
 536
 537         /* This is needed in order to enable forwarding support */
 538         if (frag_size)
 539                 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
 540                                      le16_to_cpu(cqe->pkt_len),
 541                                      le16_to_cpu(cqe->num_of_coalesced_segs));
 542
 543 #ifdef BNX2X_STOP_ON_ERROR
 544         if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
 545                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 546                           pages, cqe_idx);
 547                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 548                 bnx2x_panic();
 549                 return -EINVAL;
 550         }
 551 #endif
 552
 553         /* Run through the SGL and compose the fragmented skb */
 554         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 555                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 556
 557                 /* FW gives the indices of the SGE as if the ring is an array
 558                    (meaning that "next" element will consume 2 indices) */
 559                 if (fp->mode == TPA_MODE_GRO)
 560                         frag_len = min_t(u32, frag_size, (u32)full_page);
 561                 else /* LRO */
 562                         frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
 563
 564                 rx_pg = &fp->rx_page_ring[sge_idx];
 565                 old_rx_pg = *rx_pg;
 566
 567                 /* If we fail to allocate a substitute page, we simply stop
 568                    where we are and drop the whole packet */
 569                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 570                 if (unlikely(err)) {
 571                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 572                         return err;
 573                 }
 574
 575                 /* Unmap the page as we r going to pass it to the stack */
 576                 dma_unmap_page(&bp->pdev->dev,
 577                                dma_unmap_addr(&old_rx_pg, mapping),
 578                                SGE_PAGES, DMA_FROM_DEVICE);
 579                 /* Add one frag and update the appropriate fields in the skb */
 580                 if (fp->mode == TPA_MODE_LRO)
 581                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 582                 else { /* GRO */
 583                         int rem;
 584                         int offset = 0;
 585                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 586                                 int len = rem > gro_size ? gro_size : rem;
 587                                 skb_fill_page_desc(skb, frag_id++,
 588                                                    old_rx_pg.page, offset, len);
 589                                 if (offset)
 590                                         get_page(old_rx_pg.page);
 591                                 offset += len;
 592                         }
 593                 }
 594
 595                 skb->data_len += frag_len;
 596                 skb->truesize += SGE_PAGES;
 597                 skb->len += frag_len;
 598
 599                 frag_size -= frag_len;
 600         }
 601
 602         return 0;
 603 }
 604
 605 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 606 {
 607         if (fp->rx_frag_size)
 608                 put_page(virt_to_head_page(data));
 609         else
 610                 kfree(data);
 611 }
 612
 613 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp)
 614 {
 615         if (fp->rx_frag_size)
 616                 return netdev_alloc_frag(fp->rx_frag_size);
 617
 618         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
 619 }
 620
 621 #ifdef CONFIG_INET
 622 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
 623 {
 624         const struct iphdr *iph = ip_hdr(skb);
 625         struct tcphdr *th;
 626
 627         skb_set_transport_header(skb, sizeof(struct iphdr));
 628         th = tcp_hdr(skb);
 629
 630         th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
 631                                   iph->saddr, iph->daddr, 0);
 632 }
 633
 634 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
 635 {
 636         struct ipv6hdr *iph = ipv6_hdr(skb);
 637         struct tcphdr *th;
 638
 639         skb_set_transport_header(skb, sizeof(struct ipv6hdr));
 640         th = tcp_hdr(skb);
 641
 642         th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
 643                                   &iph->saddr, &iph->daddr, 0);
 644 }
 645
 646 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
 647                             void (*gro_func)(struct bnx2x*, struct sk_buff*))
 648 {
 649         skb_set_network_header(skb, 0);
 650         gro_func(bp, skb);
 651         tcp_gro_complete(skb);
 652 }
 653 #endif
 654
 655 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 656                                struct sk_buff *skb)
 657 {
 658 #ifdef CONFIG_INET
 659         if (skb_shinfo(skb)->gso_size) {
 660                 switch (be16_to_cpu(skb->protocol)) {
 661                 case ETH_P_IP:
 662                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
 663                         break;
 664                 case ETH_P_IPV6:
 665                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
 666                         break;
 667                 default:
 668                         BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
 669                                   be16_to_cpu(skb->protocol));
 670                 }
 671         }
 672 #endif
 673         napi_gro_receive(&fp->napi, skb);
 674 }
 675
 676 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 677                            struct bnx2x_agg_info *tpa_info,
 678                            u16 pages,
 679                            struct eth_end_agg_rx_cqe *cqe,
 680                            u16 cqe_idx)
 681 {
 682         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 683         u8 pad = tpa_info->placement_offset;
 684         u16 len = tpa_info->len_on_bd;
 685         struct sk_buff *skb = NULL;
 686         u8 *new_data, *data = rx_buf->data;
 687         u8 old_tpa_state = tpa_info->tpa_state;
 688
 689         tpa_info->tpa_state = BNX2X_TPA_STOP;
 690
 691         /* If we there was an error during the handling of the TPA_START -
 692          * drop this aggregation.
 693          */
 694         if (old_tpa_state == BNX2X_TPA_ERROR)
 695                 goto drop;
 696
 697         /* Try to allocate the new data */
 698         new_data = bnx2x_frag_alloc(fp);
 699         /* Unmap skb in the pool anyway, as we are going to change
 700            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 701            fails. */
 702         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 703                          fp->rx_buf_size, DMA_FROM_DEVICE);
 704         if (likely(new_data))
 705                 skb = build_skb(data, fp->rx_frag_size);
 706
 707         if (likely(skb)) {
 708 #ifdef BNX2X_STOP_ON_ERROR
 709                 if (pad + len > fp->rx_buf_size) {
 710                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 711                                   pad, len, fp->rx_buf_size);
 712                         bnx2x_panic();
 713                         return;
 714                 }
 715 #endif
 716
 717                 skb_reserve(skb, pad + NET_SKB_PAD);
 718                 skb_put(skb, len);
 719                 skb->rxhash = tpa_info->rxhash;
 720                 skb->l4_rxhash = tpa_info->l4_rxhash;
 721
 722                 skb->protocol = eth_type_trans(skb, bp->dev);
 723                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 724
 725                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 726                                          skb, cqe, cqe_idx)) {
 727                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 728                                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
 729                         bnx2x_gro_receive(bp, fp, skb);
 730                 } else {
 731                         DP(NETIF_MSG_RX_STATUS,
 732                            "Failed to allocate new pages - dropping packet!\n");
 733                         dev_kfree_skb_any(skb);
 734                 }
 735
 736
 737                 /* put new data in bin */
 738                 rx_buf->data = new_data;
 739
 740                 return;
 741         }
 742         bnx2x_frag_free(fp, new_data);
 743 drop:
 744         /* drop the packet and keep the buffer in the bin */
 745         DP(NETIF_MSG_RX_STATUS,
 746            "Failed to allocate or map a new skb - dropping packet!\n");
 747         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 748 }
 749
 750 static int bnx2x_alloc_rx_data(struct bnx2x *bp,
 751                                struct bnx2x_fastpath *fp, u16 index)
 752 {
 753         u8 *data;
 754         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 755         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 756         dma_addr_t mapping;
 757
 758         data = bnx2x_frag_alloc(fp);
 759         if (unlikely(data == NULL))
 760                 return -ENOMEM;
 761
 762         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 763                                  fp->rx_buf_size,
 764                                  DMA_FROM_DEVICE);
 765         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 766                 bnx2x_frag_free(fp, data);
 767                 BNX2X_ERR("Can't map rx data\n");
 768                 return -ENOMEM;
 769         }
 770
 771         rx_buf->data = data;
 772         dma_unmap_addr_set(rx_buf, mapping, mapping);
 773
 774         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 775         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 776
 777         return 0;
 778 }
 779
 780 static
 781 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 782                                  struct bnx2x_fastpath *fp,
 783                                  struct bnx2x_eth_q_stats *qstats)
 784 {
 785         /* Do nothing if no L4 csum validation was done.
 786          * We do not check whether IP csum was validated. For IPv4 we assume
 787          * that if the card got as far as validating the L4 csum, it also
 788          * validated the IP csum. IPv6 has no IP csum.
 789          */
 790         if (cqe->fast_path_cqe.status_flags &
 791             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 792                 return;
 793
 794         /* If L4 validation was done, check if an error was found. */
 795
 796         if (cqe->fast_path_cqe.type_error_flags &
 797             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 798              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 799                 qstats->hw_csum_err++;
 800         else
 801                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 802 }
 803
 804 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 805 {
 806         struct bnx2x *bp = fp->bp;
 807         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 808         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 809         int rx_pkt = 0;
 810
 811 #ifdef BNX2X_STOP_ON_ERROR
 812         if (unlikely(bp->panic))
 813                 return 0;
 814 #endif
 815
 816         /* CQ "next element" is of the size of the regular element,
 817            that's why it's ok here */
 818         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 819         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 820                 hw_comp_cons++;
 821
 822         bd_cons = fp->rx_bd_cons;
 823         bd_prod = fp->rx_bd_prod;
 824         bd_prod_fw = bd_prod;
 825         sw_comp_cons = fp->rx_comp_cons;
 826         sw_comp_prod = fp->rx_comp_prod;
 827
 828         /* Memory barrier necessary as speculative reads of the rx
 829          * buffer can be ahead of the index in the status block
 830          */
 831         rmb();
 832
 833         DP(NETIF_MSG_RX_STATUS,
 834            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 835            fp->index, hw_comp_cons, sw_comp_cons);
 836
 837         while (sw_comp_cons != hw_comp_cons) {
 838                 struct sw_rx_bd *rx_buf = NULL;
 839                 struct sk_buff *skb;
 840                 union eth_rx_cqe *cqe;
 841                 struct eth_fast_path_rx_cqe *cqe_fp;
 842                 u8 cqe_fp_flags;
 843                 enum eth_rx_cqe_type cqe_fp_type;
 844                 u16 len, pad, queue;
 845                 u8 *data;
 846                 bool l4_rxhash;
 847
 848 #ifdef BNX2X_STOP_ON_ERROR
 849                 if (unlikely(bp->panic))
 850                         return 0;
 851 #endif
 852
 853                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 854                 bd_prod = RX_BD(bd_prod);
 855                 bd_cons = RX_BD(bd_cons);
 856
 857                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 858                 cqe_fp = &cqe->fast_path_cqe;
 859                 cqe_fp_flags = cqe_fp->type_error_flags;
 860                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 861
 862                 DP(NETIF_MSG_RX_STATUS,
 863                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 864                    CQE_TYPE(cqe_fp_flags),
 865                    cqe_fp_flags, cqe_fp->status_flags,
 866                    le32_to_cpu(cqe_fp->rss_hash_result),
 867                    le16_to_cpu(cqe_fp->vlan_tag),
 868                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 869
 870                 /* is this a slowpath msg? */
 871                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 872                         bnx2x_sp_event(fp, cqe);
 873                         goto next_cqe;
 874                 }
 875
 876                 rx_buf = &fp->rx_buf_ring[bd_cons];
 877                 data = rx_buf->data;
 878
 879                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 880                         struct bnx2x_agg_info *tpa_info;
 881                         u16 frag_size, pages;
 882 #ifdef BNX2X_STOP_ON_ERROR
 883                         /* sanity check */
 884                         if (fp->disable_tpa &&
 885                             (CQE_TYPE_START(cqe_fp_type) ||
 886                              CQE_TYPE_STOP(cqe_fp_type)))
 887                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 888                                           CQE_TYPE(cqe_fp_type));
 889 #endif
 890
 891                         if (CQE_TYPE_START(cqe_fp_type)) {
 892                                 u16 queue = cqe_fp->queue_index;
 893                                 DP(NETIF_MSG_RX_STATUS,
 894                                    "calling tpa_start on queue %d\n",
 895                                    queue);
 896
 897                                 bnx2x_tpa_start(fp, queue,
 898                                                 bd_cons, bd_prod,
 899                                                 cqe_fp);
 900
 901                                 goto next_rx;
 902
 903                         }
 904                         queue = cqe->end_agg_cqe.queue_index;
 905                         tpa_info = &fp->tpa_info[queue];
 906                         DP(NETIF_MSG_RX_STATUS,
 907                            "calling tpa_stop on queue %d\n",
 908                            queue);
 909
 910                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 911                                     tpa_info->len_on_bd;
 912
 913                         if (fp->mode == TPA_MODE_GRO)
 914                                 pages = (frag_size + tpa_info->full_page - 1) /
 915                                          tpa_info->full_page;
 916                         else
 917                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 918                                         SGE_PAGE_SHIFT;
 919
 920                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 921                                        &cqe->end_agg_cqe, comp_ring_cons);
 922 #ifdef BNX2X_STOP_ON_ERROR
 923                         if (bp->panic)
 924                                 return 0;
 925 #endif
 926
 927                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 928                         goto next_cqe;
 929                 }
 930                 /* non TPA */
 931                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 932                 pad = cqe_fp->placement_offset;
 933                 dma_sync_single_for_cpu(&bp->pdev->dev,
 934                                         dma_unmap_addr(rx_buf, mapping),
 935                                         pad + RX_COPY_THRESH,
 936                                         DMA_FROM_DEVICE);
 937                 pad += NET_SKB_PAD;
 938                 prefetch(data + pad); /* speedup eth_type_trans() */
 939                 /* is this an error packet? */
 940                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 941                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 942                            "ERROR  flags %x  rx packet %u\n",
 943                            cqe_fp_flags, sw_comp_cons);
 944                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
 945                         goto reuse_rx;
 946                 }
 947
 948                 /* Since we don't have a jumbo ring
 949                  * copy small packets if mtu > 1500
 950                  */
 951                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 952                     (len <= RX_COPY_THRESH)) {
 953                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 954                         if (skb == NULL) {
 955                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 956                                    "ERROR  packet dropped because of alloc failure\n");
 957                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 958                                 goto reuse_rx;
 959                         }
 960                         memcpy(skb->data, data + pad, len);
 961                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 962                 } else {
 963                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
 964                                 dma_unmap_single(&bp->pdev->dev,
 965                                                  dma_unmap_addr(rx_buf, mapping),
 966                                                  fp->rx_buf_size,
 967                                                  DMA_FROM_DEVICE);
 968                                 skb = build_skb(data, fp->rx_frag_size);
 969                                 if (unlikely(!skb)) {
 970                                         bnx2x_frag_free(fp, data);
 971                                         bnx2x_fp_qstats(bp, fp)->
 972                                                         rx_skb_alloc_failed++;
 973                                         goto next_rx;
 974                                 }
 975                                 skb_reserve(skb, pad);
 976                         } else {
 977                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 978                                    "ERROR  packet dropped because of alloc failure\n");
 979                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 980 reuse_rx:
 981                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 982                                 goto next_rx;
 983                         }
 984                 }
 985
 986                 skb_put(skb, len);
 987                 skb->protocol = eth_type_trans(skb, bp->dev);
 988
 989                 /* Set Toeplitz hash for a none-LRO skb */
 990                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
 991                 skb->l4_rxhash = l4_rxhash;
 992
 993                 skb_checksum_none_assert(skb);
 994
 995                 if (bp->dev->features & NETIF_F_RXCSUM)
 996                         bnx2x_csum_validate(skb, cqe, fp,
 997                                             bnx2x_fp_qstats(bp, fp));
 998
 999                 skb_record_rx_queue(skb, fp->rx_queue);
1000
1001                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1002                     PARSING_FLAGS_VLAN)
1003                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1004                                                le16_to_cpu(cqe_fp->vlan_tag));
1005                 napi_gro_receive(&fp->napi, skb);
1006
1007
1008 next_rx:
1009                 rx_buf->data = NULL;
1010
1011                 bd_cons = NEXT_RX_IDX(bd_cons);
1012                 bd_prod = NEXT_RX_IDX(bd_prod);
1013                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1014                 rx_pkt++;
1015 next_cqe:
1016                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1017                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1018
1019                 if (rx_pkt == budget)
1020                         break;
1021         } /* while */
1022
1023         fp->rx_bd_cons = bd_cons;
1024         fp->rx_bd_prod = bd_prod_fw;
1025         fp->rx_comp_cons = sw_comp_cons;
1026         fp->rx_comp_prod = sw_comp_prod;
1027
1028         /* Update producers */
1029         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1030                              fp->rx_sge_prod);
1031
1032         fp->rx_pkt += rx_pkt;
1033         fp->rx_calls++;
1034
1035         return rx_pkt;
1036 }
1037
1038 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1039 {
1040         struct bnx2x_fastpath *fp = fp_cookie;
1041         struct bnx2x *bp = fp->bp;
1042         u8 cos;
1043
1044         DP(NETIF_MSG_INTR,
1045            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1046            fp->index, fp->fw_sb_id, fp->igu_sb_id);
1047
1048         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1049
1050 #ifdef BNX2X_STOP_ON_ERROR
1051         if (unlikely(bp->panic))
1052                 return IRQ_HANDLED;
1053 #endif
1054
1055         /* Handle Rx and Tx according to MSI-X vector */
1056         prefetch(fp->rx_cons_sb);
1057
1058         for_each_cos_in_tx_queue(fp, cos)
1059                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1060
1061         prefetch(&fp->sb_running_index[SM_RX_ID]);
1062         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1063
1064         return IRQ_HANDLED;
1065 }
1066
1067 /* HW Lock for shared dual port PHYs */
1068 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1069 {
1070         mutex_lock(&bp->port.phy_mutex);
1071
1072         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1073 }
1074
1075 void bnx2x_release_phy_lock(struct bnx2x *bp)
1076 {
1077         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1078
1079         mutex_unlock(&bp->port.phy_mutex);
1080 }
1081
1082 /* calculates MF speed according to current linespeed and MF configuration */
1083 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1084 {
1085         u16 line_speed = bp->link_vars.line_speed;
1086         if (IS_MF(bp)) {
1087                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1088                                                    bp->mf_config[BP_VN(bp)]);
1089
1090                 /* Calculate the current MAX line speed limit for the MF
1091                  * devices
1092                  */
1093                 if (IS_MF_SI(bp))
1094                         line_speed = (line_speed * maxCfg) / 100;
1095                 else { /* SD mode */
1096                         u16 vn_max_rate = maxCfg * 100;
1097
1098                         if (vn_max_rate < line_speed)
1099                                 line_speed = vn_max_rate;
1100                 }
1101         }
1102
1103         return line_speed;
1104 }
1105
1106 /**
1107  * bnx2x_fill_report_data - fill link report data to report
1108  *
1109  * @bp:         driver handle
1110  * @data:       link state to update
1111  *
1112  * It uses a none-atomic bit operations because is called under the mutex.
1113  */
1114 static void bnx2x_fill_report_data(struct bnx2x *bp,
1115                                    struct bnx2x_link_report_data *data)
1116 {
1117         u16 line_speed = bnx2x_get_mf_speed(bp);
1118
1119         memset(data, 0, sizeof(*data));
1120
1121         /* Fill the report data: efective line speed */
1122         data->line_speed = line_speed;
1123
1124         /* Link is down */
1125         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1126                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1127                           &data->link_report_flags);
1128
1129         /* Full DUPLEX */
1130         if (bp->link_vars.duplex == DUPLEX_FULL)
1131                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1132
1133         /* Rx Flow Control is ON */
1134         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1135                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1136
1137         /* Tx Flow Control is ON */
1138         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1139                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1140 }
1141
1142 /**
1143  * bnx2x_link_report - report link status to OS.
1144  *
1145  * @bp:         driver handle
1146  *
1147  * Calls the __bnx2x_link_report() under the same locking scheme
1148  * as a link/PHY state managing code to ensure a consistent link
1149  * reporting.
1150  */
1151
1152 void bnx2x_link_report(struct bnx2x *bp)
1153 {
1154         bnx2x_acquire_phy_lock(bp);
1155         __bnx2x_link_report(bp);
1156         bnx2x_release_phy_lock(bp);
1157 }
1158
1159 /**
1160  * __bnx2x_link_report - report link status to OS.
1161  *
1162  * @bp:         driver handle
1163  *
1164  * None atomic inmlementation.
1165  * Should be called under the phy_lock.
1166  */
1167 void __bnx2x_link_report(struct bnx2x *bp)
1168 {
1169         struct bnx2x_link_report_data cur_data;
1170
1171         /* reread mf_cfg */
1172         if (IS_PF(bp) && !CHIP_IS_E1(bp))
1173                 bnx2x_read_mf_cfg(bp);
1174
1175         /* Read the current link report info */
1176         bnx2x_fill_report_data(bp, &cur_data);
1177
1178         /* Don't report link down or exactly the same link status twice */
1179         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1180             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1181                       &bp->last_reported_link.link_report_flags) &&
1182              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1183                       &cur_data.link_report_flags)))
1184                 return;
1185
1186         bp->link_cnt++;
1187
1188         /* We are going to report a new link parameters now -
1189          * remember the current data for the next time.
1190          */
1191         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1192
1193         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1194                      &cur_data.link_report_flags)) {
1195                 netif_carrier_off(bp->dev);
1196                 netdev_err(bp->dev, "NIC Link is Down\n");
1197                 return;
1198         } else {
1199                 const char *duplex;
1200                 const char *flow;
1201
1202                 netif_carrier_on(bp->dev);
1203
1204                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1205                                        &cur_data.link_report_flags))
1206                         duplex = "full";
1207                 else
1208                         duplex = "half";
1209
1210                 /* Handle the FC at the end so that only these flags would be
1211                  * possibly set. This way we may easily check if there is no FC
1212                  * enabled.
1213                  */
1214                 if (cur_data.link_report_flags) {
1215                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1216                                      &cur_data.link_report_flags)) {
1217                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1218                                      &cur_data.link_report_flags))
1219                                         flow = "ON - receive & transmit";
1220                                 else
1221                                         flow = "ON - receive";
1222                         } else {
1223                                 flow = "ON - transmit";
1224                         }
1225                 } else {
1226                         flow = "none";
1227                 }
1228                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1229                             cur_data.line_speed, duplex, flow);
1230         }
1231 }
1232
1233 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1234 {
1235         int i;
1236
1237         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1238                 struct eth_rx_sge *sge;
1239
1240                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1241                 sge->addr_hi =
1242                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1243                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1244
1245                 sge->addr_lo =
1246                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1247                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1248         }
1249 }
1250
1251 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1252                                 struct bnx2x_fastpath *fp, int last)
1253 {
1254         int i;
1255
1256         for (i = 0; i < last; i++) {
1257                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1258                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1259                 u8 *data = first_buf->data;
1260
1261                 if (data == NULL) {
1262                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1263                         continue;
1264                 }
1265                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1266                         dma_unmap_single(&bp->pdev->dev,
1267                                          dma_unmap_addr(first_buf, mapping),
1268                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1269                 bnx2x_frag_free(fp, data);
1270                 first_buf->data = NULL;
1271         }
1272 }
1273
1274 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1275 {
1276         int j;
1277
1278         for_each_rx_queue_cnic(bp, j) {
1279                 struct bnx2x_fastpath *fp = &bp->fp[j];
1280
1281                 fp->rx_bd_cons = 0;
1282
1283                 /* Activate BD ring */
1284                 /* Warning!
1285                  * this will generate an interrupt (to the TSTORM)
1286                  * must only be done after chip is initialized
1287                  */
1288                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1289                                      fp->rx_sge_prod);
1290         }
1291 }
1292
1293 void bnx2x_init_rx_rings(struct bnx2x *bp)
1294 {
1295         int func = BP_FUNC(bp);
1296         u16 ring_prod;
1297         int i, j;
1298
1299         /* Allocate TPA resources */
1300         for_each_eth_queue(bp, j) {
1301                 struct bnx2x_fastpath *fp = &bp->fp[j];
1302
1303                 DP(NETIF_MSG_IFUP,
1304                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1305
1306                 if (!fp->disable_tpa) {
1307                         /* Fill the per-aggregtion pool */
1308                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1309                                 struct bnx2x_agg_info *tpa_info =
1310                                         &fp->tpa_info[i];
1311                                 struct sw_rx_bd *first_buf =
1312                                         &tpa_info->first_buf;
1313
1314                                 first_buf->data = bnx2x_frag_alloc(fp);
1315                                 if (!first_buf->data) {
1316                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1317                                                   j);
1318                                         bnx2x_free_tpa_pool(bp, fp, i);
1319                                         fp->disable_tpa = 1;
1320                                         break;
1321                                 }
1322                                 dma_unmap_addr_set(first_buf, mapping, 0);
1323                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1324                         }
1325
1326                         /* "next page" elements initialization */
1327                         bnx2x_set_next_page_sgl(fp);
1328
1329                         /* set SGEs bit mask */
1330                         bnx2x_init_sge_ring_bit_mask(fp);
1331
1332                         /* Allocate SGEs and initialize the ring elements */
1333                         for (i = 0, ring_prod = 0;
1334                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1335
1336                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1337                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1338                                                   i);
1339                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1340                                                   j);
1341                                         /* Cleanup already allocated elements */
1342                                         bnx2x_free_rx_sge_range(bp, fp,
1343                                                                 ring_prod);
1344                                         bnx2x_free_tpa_pool(bp, fp,
1345                                                             MAX_AGG_QS(bp));
1346                                         fp->disable_tpa = 1;
1347                                         ring_prod = 0;
1348                                         break;
1349                                 }
1350                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1351                         }
1352
1353                         fp->rx_sge_prod = ring_prod;
1354                 }
1355         }
1356
1357         for_each_eth_queue(bp, j) {
1358                 struct bnx2x_fastpath *fp = &bp->fp[j];
1359
1360                 fp->rx_bd_cons = 0;
1361
1362                 /* Activate BD ring */
1363                 /* Warning!
1364                  * this will generate an interrupt (to the TSTORM)
1365                  * must only be done after chip is initialized
1366                  */
1367                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1368                                      fp->rx_sge_prod);
1369
1370                 if (j != 0)
1371                         continue;
1372
1373                 if (CHIP_IS_E1(bp)) {
1374                         REG_WR(bp, BAR_USTRORM_INTMEM +
1375                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1376                                U64_LO(fp->rx_comp_mapping));
1377                         REG_WR(bp, BAR_USTRORM_INTMEM +
1378                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1379                                U64_HI(fp->rx_comp_mapping));
1380                 }
1381         }
1382 }
1383
1384 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1385 {
1386         u8 cos;
1387         struct bnx2x *bp = fp->bp;
1388
1389         for_each_cos_in_tx_queue(fp, cos) {
1390                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1391                 unsigned pkts_compl = 0, bytes_compl = 0;
1392
1393                 u16 sw_prod = txdata->tx_pkt_prod;
1394                 u16 sw_cons = txdata->tx_pkt_cons;
1395
1396                 while (sw_cons != sw_prod) {
1397                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1398                                           &pkts_compl, &bytes_compl);
1399                         sw_cons++;
1400                 }
1401
1402                 netdev_tx_reset_queue(
1403                         netdev_get_tx_queue(bp->dev,
1404                                             txdata->txq_index));
1405         }
1406 }
1407
1408 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1409 {
1410         int i;
1411
1412         for_each_tx_queue_cnic(bp, i) {
1413                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1414         }
1415 }
1416
1417 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1418 {
1419         int i;
1420
1421         for_each_eth_queue(bp, i) {
1422                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1423         }
1424 }
1425
1426 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1427 {
1428         struct bnx2x *bp = fp->bp;
1429         int i;
1430
1431         /* ring wasn't allocated */
1432         if (fp->rx_buf_ring == NULL)
1433                 return;
1434
1435         for (i = 0; i < NUM_RX_BD; i++) {
1436                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1437                 u8 *data = rx_buf->data;
1438
1439                 if (data == NULL)
1440                         continue;
1441                 dma_unmap_single(&bp->pdev->dev,
1442                                  dma_unmap_addr(rx_buf, mapping),
1443                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1444
1445                 rx_buf->data = NULL;
1446                 bnx2x_frag_free(fp, data);
1447         }
1448 }
1449
1450 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1451 {
1452         int j;
1453
1454         for_each_rx_queue_cnic(bp, j) {
1455                 bnx2x_free_rx_bds(&bp->fp[j]);
1456         }
1457 }
1458
1459 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1460 {
1461         int j;
1462
1463         for_each_eth_queue(bp, j) {
1464                 struct bnx2x_fastpath *fp = &bp->fp[j];
1465
1466                 bnx2x_free_rx_bds(fp);
1467
1468                 if (!fp->disable_tpa)
1469                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1470         }
1471 }
1472
1473 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1474 {
1475         bnx2x_free_tx_skbs_cnic(bp);
1476         bnx2x_free_rx_skbs_cnic(bp);
1477 }
1478
1479 void bnx2x_free_skbs(struct bnx2x *bp)
1480 {
1481         bnx2x_free_tx_skbs(bp);
1482         bnx2x_free_rx_skbs(bp);
1483 }
1484
1485 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1486 {
1487         /* load old values */
1488         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1489
1490         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1491                 /* leave all but MAX value */
1492                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1493
1494                 /* set new MAX value */
1495                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1496                                 & FUNC_MF_CFG_MAX_BW_MASK;
1497
1498                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1499         }
1500 }
1501
1502 /**
1503  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1504  *
1505  * @bp:         driver handle
1506  * @nvecs:      number of vectors to be released
1507  */
1508 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1509 {
1510         int i, offset = 0;
1511
1512         if (nvecs == offset)
1513                 return;
1514
1515         /* VFs don't have a default SB */
1516         if (IS_PF(bp)) {
1517                 free_irq(bp->msix_table[offset].vector, bp->dev);
1518                 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1519                    bp->msix_table[offset].vector);
1520                 offset++;
1521         }
1522
1523         if (CNIC_SUPPORT(bp)) {
1524                 if (nvecs == offset)
1525                         return;
1526                 offset++;
1527         }
1528
1529         for_each_eth_queue(bp, i) {
1530                 if (nvecs == offset)
1531                         return;
1532                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1533                    i, bp->msix_table[offset].vector);
1534
1535                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1536         }
1537 }
1538
1539 void bnx2x_free_irq(struct bnx2x *bp)
1540 {
1541         if (bp->flags & USING_MSIX_FLAG &&
1542             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1543                 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1544
1545                 /* vfs don't have a default status block */
1546                 if (IS_PF(bp))
1547                         nvecs++;
1548
1549                 bnx2x_free_msix_irqs(bp, nvecs);
1550         } else {
1551                 free_irq(bp->dev->irq, bp->dev);
1552         }
1553 }
1554
1555 int bnx2x_enable_msix(struct bnx2x *bp)
1556 {
1557         int msix_vec = 0, i, rc;
1558
1559         /* VFs don't have a default status block */
1560         if (IS_PF(bp)) {
1561                 bp->msix_table[msix_vec].entry = msix_vec;
1562                 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1563                                bp->msix_table[0].entry);
1564                 msix_vec++;
1565         }
1566
1567         /* Cnic requires an msix vector for itself */
1568         if (CNIC_SUPPORT(bp)) {
1569                 bp->msix_table[msix_vec].entry = msix_vec;
1570                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1571                                msix_vec, bp->msix_table[msix_vec].entry);
1572                 msix_vec++;
1573         }
1574
1575         /* We need separate vectors for ETH queues only (not FCoE) */
1576         for_each_eth_queue(bp, i) {
1577                 bp->msix_table[msix_vec].entry = msix_vec;
1578                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1579                                msix_vec, msix_vec, i);
1580                 msix_vec++;
1581         }
1582
1583         DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1584            msix_vec);
1585
1586         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec);
1587
1588         /*
1589          * reconfigure number of tx/rx queues according to available
1590          * MSI-X vectors
1591          */
1592         if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1593                 /* how less vectors we will have? */
1594                 int diff = msix_vec - rc;
1595
1596                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1597
1598                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1599
1600                 if (rc) {
1601                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1602                         goto no_msix;
1603                 }
1604                 /*
1605                  * decrease number of queues by number of unallocated entries
1606                  */
1607                 bp->num_ethernet_queues -= diff;
1608                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1609
1610                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1611                                bp->num_queues);
1612         } else if (rc > 0) {
1613                 /* Get by with single vector */
1614                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1615                 if (rc) {
1616                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1617                                        rc);
1618                         goto no_msix;
1619                 }
1620
1621                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1622                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1623
1624                 BNX2X_DEV_INFO("set number of queues to 1\n");
1625                 bp->num_ethernet_queues = 1;
1626                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1627         } else if (rc < 0) {
1628                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1629                 goto no_msix;
1630         }
1631
1632         bp->flags |= USING_MSIX_FLAG;
1633
1634         return 0;
1635
1636 no_msix:
1637         /* fall to INTx if not enough memory */
1638         if (rc == -ENOMEM)
1639                 bp->flags |= DISABLE_MSI_FLAG;
1640
1641         return rc;
1642 }
1643
1644 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1645 {
1646         int i, rc, offset = 0;
1647
1648         /* no default status block for vf */
1649         if (IS_PF(bp)) {
1650                 rc = request_irq(bp->msix_table[offset++].vector,
1651                                  bnx2x_msix_sp_int, 0,
1652                                  bp->dev->name, bp->dev);
1653                 if (rc) {
1654                         BNX2X_ERR("request sp irq failed\n");
1655                         return -EBUSY;
1656                 }
1657         }
1658
1659         if (CNIC_SUPPORT(bp))
1660                 offset++;
1661
1662         for_each_eth_queue(bp, i) {
1663                 struct bnx2x_fastpath *fp = &bp->fp[i];
1664                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1665                          bp->dev->name, i);
1666
1667                 rc = request_irq(bp->msix_table[offset].vector,
1668                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1669                 if (rc) {
1670                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1671                               bp->msix_table[offset].vector, rc);
1672                         bnx2x_free_msix_irqs(bp, offset);
1673                         return -EBUSY;
1674                 }
1675
1676                 offset++;
1677         }
1678
1679         i = BNX2X_NUM_ETH_QUEUES(bp);
1680         if (IS_PF(bp)) {
1681                 offset = 1 + CNIC_SUPPORT(bp);
1682                 netdev_info(bp->dev,
1683                             "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1684                             bp->msix_table[0].vector,
1685                             0, bp->msix_table[offset].vector,
1686                             i - 1, bp->msix_table[offset + i - 1].vector);
1687         } else {
1688                 offset = CNIC_SUPPORT(bp);
1689                 netdev_info(bp->dev,
1690                             "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1691                             0, bp->msix_table[offset].vector,
1692                             i - 1, bp->msix_table[offset + i - 1].vector);
1693         }
1694         return 0;
1695 }
1696
1697 int bnx2x_enable_msi(struct bnx2x *bp)
1698 {
1699         int rc;
1700
1701         rc = pci_enable_msi(bp->pdev);
1702         if (rc) {
1703                 BNX2X_DEV_INFO("MSI is not attainable\n");
1704                 return -1;
1705         }
1706         bp->flags |= USING_MSI_FLAG;
1707
1708         return 0;
1709 }
1710
1711 static int bnx2x_req_irq(struct bnx2x *bp)
1712 {
1713         unsigned long flags;
1714         unsigned int irq;
1715
1716         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1717                 flags = 0;
1718         else
1719                 flags = IRQF_SHARED;
1720
1721         if (bp->flags & USING_MSIX_FLAG)
1722                 irq = bp->msix_table[0].vector;
1723         else
1724                 irq = bp->pdev->irq;
1725
1726         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1727 }
1728
1729 int bnx2x_setup_irqs(struct bnx2x *bp)
1730 {
1731         int rc = 0;
1732         if (bp->flags & USING_MSIX_FLAG &&
1733             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1734                 rc = bnx2x_req_msix_irqs(bp);
1735                 if (rc)
1736                         return rc;
1737         } else {
1738                 rc = bnx2x_req_irq(bp);
1739                 if (rc) {
1740                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1741                         return rc;
1742                 }
1743                 if (bp->flags & USING_MSI_FLAG) {
1744                         bp->dev->irq = bp->pdev->irq;
1745                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1746                                     bp->dev->irq);
1747                 }
1748                 if (bp->flags & USING_MSIX_FLAG) {
1749                         bp->dev->irq = bp->msix_table[0].vector;
1750                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1751                                     bp->dev->irq);
1752                 }
1753         }
1754
1755         return 0;
1756 }
1757
1758 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1759 {
1760         int i;
1761
1762         for_each_rx_queue_cnic(bp, i)
1763                 napi_enable(&bnx2x_fp(bp, i, napi));
1764 }
1765
1766 static void bnx2x_napi_enable(struct bnx2x *bp)
1767 {
1768         int i;
1769
1770         for_each_eth_queue(bp, i)
1771                 napi_enable(&bnx2x_fp(bp, i, napi));
1772 }
1773
1774 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1775 {
1776         int i;
1777
1778         for_each_rx_queue_cnic(bp, i)
1779                 napi_disable(&bnx2x_fp(bp, i, napi));
1780 }
1781
1782 static void bnx2x_napi_disable(struct bnx2x *bp)
1783 {
1784         int i;
1785
1786         for_each_eth_queue(bp, i)
1787                 napi_disable(&bnx2x_fp(bp, i, napi));
1788 }
1789
1790 void bnx2x_netif_start(struct bnx2x *bp)
1791 {
1792         if (netif_running(bp->dev)) {
1793                 bnx2x_napi_enable(bp);
1794                 if (CNIC_LOADED(bp))
1795                         bnx2x_napi_enable_cnic(bp);
1796                 bnx2x_int_enable(bp);
1797                 if (bp->state == BNX2X_STATE_OPEN)
1798                         netif_tx_wake_all_queues(bp->dev);
1799         }
1800 }
1801
1802 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1803 {
1804         bnx2x_int_disable_sync(bp, disable_hw);
1805         bnx2x_napi_disable(bp);
1806         if (CNIC_LOADED(bp))
1807                 bnx2x_napi_disable_cnic(bp);
1808 }
1809
1810 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1811 {
1812         struct bnx2x *bp = netdev_priv(dev);
1813
1814         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1815                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1816                 u16 ether_type = ntohs(hdr->h_proto);
1817
1818                 /* Skip VLAN tag if present */
1819                 if (ether_type == ETH_P_8021Q) {
1820                         struct vlan_ethhdr *vhdr =
1821                                 (struct vlan_ethhdr *)skb->data;
1822
1823                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1824                 }
1825
1826                 /* If ethertype is FCoE or FIP - use FCoE ring */
1827                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1828                         return bnx2x_fcoe_tx(bp, txq_index);
1829         }
1830
1831         /* select a non-FCoE queue */
1832         return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1833 }
1834
1835 void bnx2x_set_num_queues(struct bnx2x *bp)
1836 {
1837         /* RSS queues */
1838         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1839
1840         /* override in STORAGE SD modes */
1841         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1842                 bp->num_ethernet_queues = 1;
1843
1844         /* Add special queues */
1845         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1846         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1847
1848         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1849 }
1850
1851 /**
1852  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1853  *
1854  * @bp:         Driver handle
1855  *
1856  * We currently support for at most 16 Tx queues for each CoS thus we will
1857  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1858  * bp->max_cos.
1859  *
1860  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1861  * index after all ETH L2 indices.
1862  *
1863  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1864  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1865  * 16..31,...) with indicies that are not coupled with any real Tx queue.
1866  *
1867  * The proper configuration of skb->queue_mapping is handled by
1868  * bnx2x_select_queue() and __skb_tx_hash().
1869  *
1870  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1871  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1872  */
1873 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1874 {
1875         int rc, tx, rx;
1876
1877         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1878         rx = BNX2X_NUM_ETH_QUEUES(bp);
1879
1880 /* account for fcoe queue */
1881         if (include_cnic && !NO_FCOE(bp)) {
1882                 rx++;
1883                 tx++;
1884         }
1885
1886         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1887         if (rc) {
1888                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1889                 return rc;
1890         }
1891         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1892         if (rc) {
1893                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1894                 return rc;
1895         }
1896
1897         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1898                           tx, rx);
1899
1900         return rc;
1901 }
1902
1903 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1904 {
1905         int i;
1906
1907         for_each_queue(bp, i) {
1908                 struct bnx2x_fastpath *fp = &bp->fp[i];
1909                 u32 mtu;
1910
1911                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1912                 if (IS_FCOE_IDX(i))
1913                         /*
1914                          * Although there are no IP frames expected to arrive to
1915                          * this ring we still want to add an
1916                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1917                          * overrun attack.
1918                          */
1919                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1920                 else
1921                         mtu = bp->dev->mtu;
1922                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1923                                   IP_HEADER_ALIGNMENT_PADDING +
1924                                   ETH_OVREHEAD +
1925                                   mtu +
1926                                   BNX2X_FW_RX_ALIGN_END;
1927                 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1928                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1929                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1930                 else
1931                         fp->rx_frag_size = 0;
1932         }
1933 }
1934
1935 static int bnx2x_init_rss_pf(struct bnx2x *bp)
1936 {
1937         int i;
1938         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1939
1940         /* Prepare the initial contents fo the indirection table if RSS is
1941          * enabled
1942          */
1943         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1944                 bp->rss_conf_obj.ind_table[i] =
1945                         bp->fp->cl_id +
1946                         ethtool_rxfh_indir_default(i, num_eth_queues);
1947
1948         /*
1949          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1950          * per-port, so if explicit configuration is needed , do it only
1951          * for a PMF.
1952          *
1953          * For 57712 and newer on the other hand it's a per-function
1954          * configuration.
1955          */
1956         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1957 }
1958
1959 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1960                         bool config_hash)
1961 {
1962         struct bnx2x_config_rss_params params = {NULL};
1963
1964         /* Although RSS is meaningless when there is a single HW queue we
1965          * still need it enabled in order to have HW Rx hash generated.
1966          *
1967          * if (!is_eth_multi(bp))
1968          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1969          */
1970
1971         params.rss_obj = rss_obj;
1972
1973         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1974
1975         __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1976
1977         /* RSS configuration */
1978         __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1979         __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1980         __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1981         __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1982         if (rss_obj->udp_rss_v4)
1983                 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
1984         if (rss_obj->udp_rss_v6)
1985                 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
1986
1987         /* Hash bits */
1988         params.rss_result_mask = MULTI_MASK;
1989
1990         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
1991
1992         if (config_hash) {
1993                 /* RSS keys */
1994                 prandom_bytes(params.rss_key, sizeof(params.rss_key));
1995                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1996         }
1997
1998         return bnx2x_config_rss(bp, &params);
1999 }
2000
2001 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2002 {
2003         struct bnx2x_func_state_params func_params = {NULL};
2004
2005         /* Prepare parameters for function state transitions */
2006         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2007
2008         func_params.f_obj = &bp->func_obj;
2009         func_params.cmd = BNX2X_F_CMD_HW_INIT;
2010
2011         func_params.params.hw_init.load_phase = load_code;
2012
2013         return bnx2x_func_state_change(bp, &func_params);
2014 }
2015
2016 /*
2017  * Cleans the object that have internal lists without sending
2018  * ramrods. Should be run when interrutps are disabled.
2019  */
2020 void bnx2x_squeeze_objects(struct bnx2x *bp)
2021 {
2022         int rc;
2023         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2024         struct bnx2x_mcast_ramrod_params rparam = {NULL};
2025         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2026
2027         /***************** Cleanup MACs' object first *************************/
2028
2029         /* Wait for completion of requested */
2030         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2031         /* Perform a dry cleanup */
2032         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2033
2034         /* Clean ETH primary MAC */
2035         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2036         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2037                                  &ramrod_flags);
2038         if (rc != 0)
2039                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2040
2041         /* Cleanup UC list */
2042         vlan_mac_flags = 0;
2043         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2044         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2045                                  &ramrod_flags);
2046         if (rc != 0)
2047                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2048
2049         /***************** Now clean mcast object *****************************/
2050         rparam.mcast_obj = &bp->mcast_obj;
2051         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2052
2053         /* Add a DEL command... */
2054         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2055         if (rc < 0)
2056                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2057                           rc);
2058
2059         /* ...and wait until all pending commands are cleared */
2060         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2061         while (rc != 0) {
2062                 if (rc < 0) {
2063                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2064                                   rc);
2065                         return;
2066                 }
2067
2068                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2069         }
2070 }
2071
2072 #ifndef BNX2X_STOP_ON_ERROR
2073 #define LOAD_ERROR_EXIT(bp, label) \
2074         do { \
2075                 (bp)->state = BNX2X_STATE_ERROR; \
2076                 goto label; \
2077         } while (0)
2078
2079 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2080         do { \
2081                 bp->cnic_loaded = false; \
2082                 goto label; \
2083         } while (0)
2084 #else /*BNX2X_STOP_ON_ERROR*/
2085 #define LOAD_ERROR_EXIT(bp, label) \
2086         do { \
2087                 (bp)->state = BNX2X_STATE_ERROR; \
2088                 (bp)->panic = 1; \
2089                 return -EBUSY; \
2090         } while (0)
2091 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2092         do { \
2093                 bp->cnic_loaded = false; \
2094                 (bp)->panic = 1; \
2095                 return -EBUSY; \
2096         } while (0)
2097 #endif /*BNX2X_STOP_ON_ERROR*/
2098
2099 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2100 {
2101         BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2102                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2103         return;
2104 }
2105
2106 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2107 {
2108         int num_groups, vf_headroom = 0;
2109         int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2110
2111         /* number of queues for statistics is number of eth queues + FCoE */
2112         u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2113
2114         /* Total number of FW statistics requests =
2115          * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2116          * and fcoe l2 queue) stats + num of queues (which includes another 1
2117          * for fcoe l2 queue if applicable)
2118          */
2119         bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2120
2121         /* vf stats appear in the request list, but their data is allocated by
2122          * the VFs themselves. We don't include them in the bp->fw_stats_num as
2123          * it is used to determine where to place the vf stats queries in the
2124          * request struct
2125          */
2126         if (IS_SRIOV(bp))
2127                 vf_headroom = bnx2x_vf_headroom(bp);
2128
2129         /* Request is built from stats_query_header and an array of
2130          * stats_query_cmd_group each of which contains
2131          * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2132          * configured in the stats_query_header.
2133          */
2134         num_groups =
2135                 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2136                  (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2137                  1 : 0));
2138
2139         DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2140            bp->fw_stats_num, vf_headroom, num_groups);
2141         bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2142                 num_groups * sizeof(struct stats_query_cmd_group);
2143
2144         /* Data for statistics requests + stats_counter
2145          * stats_counter holds per-STORM counters that are incremented
2146          * when STORM has finished with the current request.
2147          * memory for FCoE offloaded statistics are counted anyway,
2148          * even if they will not be sent.
2149          * VF stats are not accounted for here as the data of VF stats is stored
2150          * in memory allocated by the VF, not here.
2151          */
2152         bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2153                 sizeof(struct per_pf_stats) +
2154                 sizeof(struct fcoe_statistics_params) +
2155                 sizeof(struct per_queue_stats) * num_queue_stats +
2156                 sizeof(struct stats_counter);
2157
2158         BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping,
2159                         bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2160
2161         /* Set shortcuts */
2162         bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2163         bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2164         bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2165                 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2166         bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2167                 bp->fw_stats_req_sz;
2168
2169         DP(BNX2X_MSG_SP, "statistics request base address set to %x %x",
2170            U64_HI(bp->fw_stats_req_mapping),
2171            U64_LO(bp->fw_stats_req_mapping));
2172         DP(BNX2X_MSG_SP, "statistics data base address set to %x %x",
2173            U64_HI(bp->fw_stats_data_mapping),
2174            U64_LO(bp->fw_stats_data_mapping));
2175         return 0;
2176
2177 alloc_mem_err:
2178         bnx2x_free_fw_stats_mem(bp);
2179         BNX2X_ERR("Can't allocate FW stats memory\n");
2180         return -ENOMEM;
2181 }
2182
2183 /* send load request to mcp and analyze response */
2184 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2185 {
2186         /* init fw_seq */
2187         bp->fw_seq =
2188                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2189                  DRV_MSG_SEQ_NUMBER_MASK);
2190         BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2191
2192         /* Get current FW pulse sequence */
2193         bp->fw_drv_pulse_wr_seq =
2194                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2195                  DRV_PULSE_SEQ_MASK);
2196         BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2197
2198         /* load request */
2199         (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ,
2200                                         DRV_MSG_CODE_LOAD_REQ_WITH_LFA);
2201
2202         /* if mcp fails to respond we must abort */
2203         if (!(*load_code)) {
2204                 BNX2X_ERR("MCP response failure, aborting\n");
2205                 return -EBUSY;
2206         }
2207
2208         /* If mcp refused (e.g. other port is in diagnostic mode) we
2209          * must abort
2210          */
2211         if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2212                 BNX2X_ERR("MCP refused load request, aborting\n");
2213                 return -EBUSY;
2214         }
2215         return 0;
2216 }
2217
2218 /* check whether another PF has already loaded FW to chip. In
2219  * virtualized environments a pf from another VM may have already
2220  * initialized the device including loading FW
2221  */
2222 int bnx2x_nic_load_analyze_req(struct bnx2x *bp, u32 load_code)
2223 {
2224         /* is another pf loaded on this engine? */
2225         if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2226             load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2227                 /* build my FW version dword */
2228                 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2229                         (BCM_5710_FW_MINOR_VERSION << 8) +
2230                         (BCM_5710_FW_REVISION_VERSION << 16) +
2231                         (BCM_5710_FW_ENGINEERING_VERSION << 24);
2232
2233                 /* read loaded FW from chip */
2234                 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2235
2236                 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2237                    loaded_fw, my_fw);
2238
2239                 /* abort nic load if version mismatch */
2240                 if (my_fw != loaded_fw) {
2241                         BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. aborting\n",
2242                                   loaded_fw, my_fw);
2243                         return -EBUSY;
2244                 }
2245         }
2246         return 0;
2247 }
2248
2249 /* returns the "mcp load_code" according to global load_count array */
2250 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2251 {
2252         int path = BP_PATH(bp);
2253
2254         DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2255            path, load_count[path][0], load_count[path][1],
2256            load_count[path][2]);
2257         load_count[path][0]++;
2258         load_count[path][1 + port]++;
2259         DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2260            path, load_count[path][0], load_count[path][1],
2261            load_count[path][2]);
2262         if (load_count[path][0] == 1)
2263                 return FW_MSG_CODE_DRV_LOAD_COMMON;
2264         else if (load_count[path][1 + port] == 1)
2265                 return FW_MSG_CODE_DRV_LOAD_PORT;
2266         else
2267                 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2268 }
2269
2270 /* mark PMF if applicable */
2271 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2272 {
2273         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2274             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2275             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2276                 bp->port.pmf = 1;
2277                 /* We need the barrier to ensure the ordering between the
2278                  * writing to bp->port.pmf here and reading it from the
2279                  * bnx2x_periodic_task().
2280                  */
2281                 smp_mb();
2282         } else {
2283                 bp->port.pmf = 0;
2284         }
2285
2286         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2287 }
2288
2289 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2290 {
2291         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2292              (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2293             (bp->common.shmem2_base)) {
2294                 if (SHMEM2_HAS(bp, dcc_support))
2295                         SHMEM2_WR(bp, dcc_support,
2296                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2297                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2298                 if (SHMEM2_HAS(bp, afex_driver_support))
2299                         SHMEM2_WR(bp, afex_driver_support,
2300                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2301         }
2302
2303         /* Set AFEX default VLAN tag to an invalid value */
2304         bp->afex_def_vlan_tag = -1;
2305 }
2306
2307 /**
2308  * bnx2x_bz_fp - zero content of the fastpath structure.
2309  *
2310  * @bp:         driver handle
2311  * @index:      fastpath index to be zeroed
2312  *
2313  * Makes sure the contents of the bp->fp[index].napi is kept
2314  * intact.
2315  */
2316 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2317 {
2318         struct bnx2x_fastpath *fp = &bp->fp[index];
2319
2320         int cos;
2321         struct napi_struct orig_napi = fp->napi;
2322         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2323         /* bzero bnx2x_fastpath contents */
2324         if (fp->tpa_info)
2325                 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2326                        sizeof(struct bnx2x_agg_info));
2327         memset(fp, 0, sizeof(*fp));
2328
2329         /* Restore the NAPI object as it has been already initialized */
2330         fp->napi = orig_napi;
2331         fp->tpa_info = orig_tpa_info;
2332         fp->bp = bp;
2333         fp->index = index;
2334         if (IS_ETH_FP(fp))
2335                 fp->max_cos = bp->max_cos;
2336         else
2337                 /* Special queues support only one CoS */
2338                 fp->max_cos = 1;
2339
2340         /* Init txdata pointers */
2341         if (IS_FCOE_FP(fp))
2342                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2343         if (IS_ETH_FP(fp))
2344                 for_each_cos_in_tx_queue(fp, cos)
2345                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2346                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2347
2348         /*
2349          * set the tpa flag for each queue. The tpa flag determines the queue
2350          * minimal size so it must be set prior to queue memory allocation
2351          */
2352         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2353                                   (bp->flags & GRO_ENABLE_FLAG &&
2354                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2355         if (bp->flags & TPA_ENABLE_FLAG)
2356                 fp->mode = TPA_MODE_LRO;
2357         else if (bp->flags & GRO_ENABLE_FLAG)
2358                 fp->mode = TPA_MODE_GRO;
2359
2360         /* We don't want TPA on an FCoE L2 ring */
2361         if (IS_FCOE_FP(fp))
2362                 fp->disable_tpa = 1;
2363 }
2364
2365 int bnx2x_load_cnic(struct bnx2x *bp)
2366 {
2367         int i, rc, port = BP_PORT(bp);
2368
2369         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2370
2371         mutex_init(&bp->cnic_mutex);
2372
2373         if (IS_PF(bp)) {
2374                 rc = bnx2x_alloc_mem_cnic(bp);
2375                 if (rc) {
2376                         BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2377                         LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2378                 }
2379         }
2380
2381         rc = bnx2x_alloc_fp_mem_cnic(bp);
2382         if (rc) {
2383                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2384                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2385         }
2386
2387         /* Update the number of queues with the cnic queues */
2388         rc = bnx2x_set_real_num_queues(bp, 1);
2389         if (rc) {
2390                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2391                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2392         }
2393
2394         /* Add all CNIC NAPI objects */
2395         bnx2x_add_all_napi_cnic(bp);
2396         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2397         bnx2x_napi_enable_cnic(bp);
2398
2399         rc = bnx2x_init_hw_func_cnic(bp);
2400         if (rc)
2401                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2402
2403         bnx2x_nic_init_cnic(bp);
2404
2405         if (IS_PF(bp)) {
2406                 /* Enable Timer scan */
2407                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2408
2409                 /* setup cnic queues */
2410                 for_each_cnic_queue(bp, i) {
2411                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2412                         if (rc) {
2413                                 BNX2X_ERR("Queue setup failed\n");
2414                                 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2415                         }
2416                 }
2417         }
2418
2419         /* Initialize Rx filter. */
2420         netif_addr_lock_bh(bp->dev);
2421         bnx2x_set_rx_mode(bp->dev);
2422         netif_addr_unlock_bh(bp->dev);
2423
2424         /* re-read iscsi info */
2425         bnx2x_get_iscsi_info(bp);
2426         bnx2x_setup_cnic_irq_info(bp);
2427         bnx2x_setup_cnic_info(bp);
2428         bp->cnic_loaded = true;
2429         if (bp->state == BNX2X_STATE_OPEN)
2430                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2431
2432
2433         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2434
2435         return 0;
2436
2437 #ifndef BNX2X_STOP_ON_ERROR
2438 load_error_cnic2:
2439         /* Disable Timer scan */
2440         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2441
2442 load_error_cnic1:
2443         bnx2x_napi_disable_cnic(bp);
2444         /* Update the number of queues without the cnic queues */
2445         rc = bnx2x_set_real_num_queues(bp, 0);
2446         if (rc)
2447                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2448 load_error_cnic0:
2449         BNX2X_ERR("CNIC-related load failed\n");
2450         bnx2x_free_fp_mem_cnic(bp);
2451         bnx2x_free_mem_cnic(bp);
2452         return rc;
2453 #endif /* ! BNX2X_STOP_ON_ERROR */
2454 }
2455
2456 /* must be called with rtnl_lock */
2457 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2458 {
2459         int port = BP_PORT(bp);
2460         int i, rc = 0, load_code = 0;
2461
2462         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2463         DP(NETIF_MSG_IFUP,
2464            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2465
2466 #ifdef BNX2X_STOP_ON_ERROR
2467         if (unlikely(bp->panic)) {
2468                 BNX2X_ERR("Can't load NIC when there is panic\n");
2469                 return -EPERM;
2470         }
2471 #endif
2472
2473         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2474
2475         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2476         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2477                 &bp->last_reported_link.link_report_flags);
2478
2479         if (IS_PF(bp))
2480                 /* must be called before memory allocation and HW init */
2481                 bnx2x_ilt_set_info(bp);
2482
2483         /*
2484          * Zero fastpath structures preserving invariants like napi, which are
2485          * allocated only once, fp index, max_cos, bp pointer.
2486          * Also set fp->disable_tpa and txdata_ptr.
2487          */
2488         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2489         for_each_queue(bp, i)
2490                 bnx2x_bz_fp(bp, i);
2491         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2492                                   bp->num_cnic_queues) *
2493                                   sizeof(struct bnx2x_fp_txdata));
2494
2495         bp->fcoe_init = false;
2496
2497         /* Set the receive queues buffer size */
2498         bnx2x_set_rx_buf_size(bp);
2499
2500         if (IS_PF(bp)) {
2501                 rc = bnx2x_alloc_mem(bp);
2502                 if (rc) {
2503                         BNX2X_ERR("Unable to allocate bp memory\n");
2504                         return rc;
2505                 }
2506         }
2507
2508         /* Allocated memory for FW statistics  */
2509         if (bnx2x_alloc_fw_stats_mem(bp))
2510                 LOAD_ERROR_EXIT(bp, load_error0);
2511
2512         /* need to be done after alloc mem, since it's self adjusting to amount
2513          * of memory available for RSS queues
2514          */
2515         rc = bnx2x_alloc_fp_mem(bp);
2516         if (rc) {
2517                 BNX2X_ERR("Unable to allocate memory for fps\n");
2518                 LOAD_ERROR_EXIT(bp, load_error0);
2519         }
2520
2521         /* request pf to initialize status blocks */
2522         if (IS_VF(bp)) {
2523                 rc = bnx2x_vfpf_init(bp);
2524                 if (rc)
2525                         LOAD_ERROR_EXIT(bp, load_error0);
2526         }
2527
2528         /* As long as bnx2x_alloc_mem() may possibly update
2529          * bp->num_queues, bnx2x_set_real_num_queues() should always
2530          * come after it. At this stage cnic queues are not counted.
2531          */
2532         rc = bnx2x_set_real_num_queues(bp, 0);
2533         if (rc) {
2534                 BNX2X_ERR("Unable to set real_num_queues\n");
2535                 LOAD_ERROR_EXIT(bp, load_error0);
2536         }
2537
2538         /* configure multi cos mappings in kernel.
2539          * this configuration may be overriden by a multi class queue discipline
2540          * or by a dcbx negotiation result.
2541          */
2542         bnx2x_setup_tc(bp->dev, bp->max_cos);
2543
2544         /* Add all NAPI objects */
2545         bnx2x_add_all_napi(bp);
2546         DP(NETIF_MSG_IFUP, "napi added\n");
2547         bnx2x_napi_enable(bp);
2548
2549         if (IS_PF(bp)) {
2550                 /* set pf load just before approaching the MCP */
2551                 bnx2x_set_pf_load(bp);
2552
2553                 /* if mcp exists send load request and analyze response */
2554                 if (!BP_NOMCP(bp)) {
2555                         /* attempt to load pf */
2556                         rc = bnx2x_nic_load_request(bp, &load_code);
2557                         if (rc)
2558                                 LOAD_ERROR_EXIT(bp, load_error1);
2559
2560                         /* what did mcp say? */
2561                         rc = bnx2x_nic_load_analyze_req(bp, load_code);
2562                         if (rc) {
2563                                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2564                                 LOAD_ERROR_EXIT(bp, load_error2);
2565                         }
2566                 } else {
2567                         load_code = bnx2x_nic_load_no_mcp(bp, port);
2568                 }
2569
2570                 /* mark pmf if applicable */
2571                 bnx2x_nic_load_pmf(bp, load_code);
2572
2573                 /* Init Function state controlling object */
2574                 bnx2x__init_func_obj(bp);
2575
2576                 /* Initialize HW */
2577                 rc = bnx2x_init_hw(bp, load_code);
2578                 if (rc) {
2579                         BNX2X_ERR("HW init failed, aborting\n");
2580                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2581                         LOAD_ERROR_EXIT(bp, load_error2);
2582                 }
2583         }
2584
2585         bnx2x_pre_irq_nic_init(bp);
2586
2587         /* Connect to IRQs */
2588         rc = bnx2x_setup_irqs(bp);
2589         if (rc) {
2590                 BNX2X_ERR("setup irqs failed\n");
2591                 if (IS_PF(bp))
2592                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2593                 LOAD_ERROR_EXIT(bp, load_error2);
2594         }
2595
2596         /* Init per-function objects */
2597         if (IS_PF(bp)) {
2598                 /* Setup NIC internals and enable interrupts */
2599                 bnx2x_post_irq_nic_init(bp, load_code);
2600
2601                 bnx2x_init_bp_objs(bp);
2602                 bnx2x_iov_nic_init(bp);
2603
2604                 /* Set AFEX default VLAN tag to an invalid value */
2605                 bp->afex_def_vlan_tag = -1;
2606                 bnx2x_nic_load_afex_dcc(bp, load_code);
2607                 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2608                 rc = bnx2x_func_start(bp);
2609                 if (rc) {
2610                         BNX2X_ERR("Function start failed!\n");
2611                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2612
2613                         LOAD_ERROR_EXIT(bp, load_error3);
2614                 }
2615
2616                 /* Send LOAD_DONE command to MCP */
2617                 if (!BP_NOMCP(bp)) {
2618                         load_code = bnx2x_fw_command(bp,
2619                                                      DRV_MSG_CODE_LOAD_DONE, 0);
2620                         if (!load_code) {
2621                                 BNX2X_ERR("MCP response failure, aborting\n");
2622                                 rc = -EBUSY;
2623                                 LOAD_ERROR_EXIT(bp, load_error3);
2624                         }
2625                 }
2626
2627                 /* initialize FW coalescing state machines in RAM */
2628                 bnx2x_update_coalesce(bp);
2629
2630                 /* setup the leading queue */
2631                 rc = bnx2x_setup_leading(bp);
2632                 if (rc) {
2633                         BNX2X_ERR("Setup leading failed!\n");
2634                         LOAD_ERROR_EXIT(bp, load_error3);
2635                 }
2636
2637                 /* set up the rest of the queues */
2638                 for_each_nondefault_eth_queue(bp, i) {
2639                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2640                         if (rc) {
2641                                 BNX2X_ERR("Queue setup failed\n");
2642                                 LOAD_ERROR_EXIT(bp, load_error3);
2643                         }
2644                 }
2645
2646                 /* setup rss */
2647                 rc = bnx2x_init_rss_pf(bp);
2648                 if (rc) {
2649                         BNX2X_ERR("PF RSS init failed\n");
2650                         LOAD_ERROR_EXIT(bp, load_error3);
2651                 }
2652
2653         } else { /* vf */
2654                 for_each_eth_queue(bp, i) {
2655                         rc = bnx2x_vfpf_setup_q(bp, i);
2656                         if (rc) {
2657                                 BNX2X_ERR("Queue setup failed\n");
2658                                 LOAD_ERROR_EXIT(bp, load_error3);
2659                         }
2660                 }
2661         }
2662
2663         /* Now when Clients are configured we are ready to work */
2664         bp->state = BNX2X_STATE_OPEN;
2665
2666         /* Configure a ucast MAC */
2667         if (IS_PF(bp))
2668                 rc = bnx2x_set_eth_mac(bp, true);
2669         else /* vf */
2670                 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2671                                            true);
2672         if (rc) {
2673                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2674                 LOAD_ERROR_EXIT(bp, load_error3);
2675         }
2676
2677         if (IS_PF(bp) && bp->pending_max) {
2678                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2679                 bp->pending_max = 0;
2680         }
2681
2682         if (bp->port.pmf) {
2683                 rc = bnx2x_initial_phy_init(bp, load_mode);
2684                 if (rc)
2685                         LOAD_ERROR_EXIT(bp, load_error3);
2686         }
2687         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2688
2689         /* Start fast path */
2690
2691         /* Initialize Rx filter. */
2692         netif_addr_lock_bh(bp->dev);
2693         bnx2x_set_rx_mode(bp->dev);
2694         netif_addr_unlock_bh(bp->dev);
2695
2696         /* Start the Tx */
2697         switch (load_mode) {
2698         case LOAD_NORMAL:
2699                 /* Tx queue should be only reenabled */
2700                 netif_tx_wake_all_queues(bp->dev);
2701                 break;
2702
2703         case LOAD_OPEN:
2704                 netif_tx_start_all_queues(bp->dev);
2705                 smp_mb__after_clear_bit();
2706                 break;
2707
2708         case LOAD_DIAG:
2709         case LOAD_LOOPBACK_EXT:
2710                 bp->state = BNX2X_STATE_DIAG;
2711                 break;
2712
2713         default:
2714                 break;
2715         }
2716
2717         if (bp->port.pmf)
2718                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2719         else
2720                 bnx2x__link_status_update(bp);
2721
2722         /* start the timer */
2723         mod_timer(&bp->timer, jiffies + bp->current_interval);
2724
2725         if (CNIC_ENABLED(bp))
2726                 bnx2x_load_cnic(bp);
2727
2728         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2729                 /* mark driver is loaded in shmem2 */
2730                 u32 val;
2731                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2732                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2733                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2734                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2735         }
2736
2737         /* Wait for all pending SP commands to complete */
2738         if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2739                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2740                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2741                 return -EBUSY;
2742         }
2743
2744         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2745         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2746                 bnx2x_dcbx_init(bp, false);
2747
2748         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2749
2750         return 0;
2751
2752 #ifndef BNX2X_STOP_ON_ERROR
2753 load_error3:
2754         if (IS_PF(bp)) {
2755                 bnx2x_int_disable_sync(bp, 1);
2756
2757                 /* Clean queueable objects */
2758                 bnx2x_squeeze_objects(bp);
2759         }
2760
2761         /* Free SKBs, SGEs, TPA pool and driver internals */
2762         bnx2x_free_skbs(bp);
2763         for_each_rx_queue(bp, i)
2764                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2765
2766         /* Release IRQs */
2767         bnx2x_free_irq(bp);
2768 load_error2:
2769         if (IS_PF(bp) && !BP_NOMCP(bp)) {
2770                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2771                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2772         }
2773
2774         bp->port.pmf = 0;
2775 load_error1:
2776         bnx2x_napi_disable(bp);
2777         bnx2x_del_all_napi(bp);
2778
2779         /* clear pf_load status, as it was already set */
2780         if (IS_PF(bp))
2781                 bnx2x_clear_pf_load(bp);
2782 load_error0:
2783         bnx2x_free_fp_mem(bp);
2784         bnx2x_free_fw_stats_mem(bp);
2785         bnx2x_free_mem(bp);
2786
2787         return rc;
2788 #endif /* ! BNX2X_STOP_ON_ERROR */
2789 }
2790
2791 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2792 {
2793         u8 rc = 0, cos, i;
2794
2795         /* Wait until tx fastpath tasks complete */
2796         for_each_tx_queue(bp, i) {
2797                 struct bnx2x_fastpath *fp = &bp->fp[i];
2798
2799                 for_each_cos_in_tx_queue(fp, cos)
2800                         rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2801                 if (rc)
2802                         return rc;
2803         }
2804         return 0;
2805 }
2806
2807 /* must be called with rtnl_lock */
2808 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2809 {
2810         int i;
2811         bool global = false;
2812
2813         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2814
2815         /* mark driver is unloaded in shmem2 */
2816         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2817                 u32 val;
2818                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2819                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2820                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2821         }
2822
2823         if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2824             (bp->state == BNX2X_STATE_CLOSED ||
2825              bp->state == BNX2X_STATE_ERROR)) {
2826                 /* We can get here if the driver has been unloaded
2827                  * during parity error recovery and is either waiting for a
2828                  * leader to complete or for other functions to unload and
2829                  * then ifdown has been issued. In this case we want to
2830                  * unload and let other functions to complete a recovery
2831                  * process.
2832                  */
2833                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2834                 bp->is_leader = 0;
2835                 bnx2x_release_leader_lock(bp);
2836                 smp_mb();
2837
2838                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2839                 BNX2X_ERR("Can't unload in closed or error state\n");
2840                 return -EINVAL;
2841         }
2842
2843         /* Nothing to do during unload if previous bnx2x_nic_load()
2844          * have not completed succesfully - all resourses are released.
2845          *
2846          * we can get here only after unsuccessful ndo_* callback, during which
2847          * dev->IFF_UP flag is still on.
2848          */
2849         if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2850                 return 0;
2851
2852         /* It's important to set the bp->state to the value different from
2853          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2854          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2855          */
2856         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2857         smp_mb();
2858
2859         if (CNIC_LOADED(bp))
2860                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2861
2862         /* Stop Tx */
2863         bnx2x_tx_disable(bp);
2864         netdev_reset_tc(bp->dev);
2865
2866         bp->rx_mode = BNX2X_RX_MODE_NONE;
2867
2868         del_timer_sync(&bp->timer);
2869
2870         if (IS_PF(bp)) {
2871                 /* Set ALWAYS_ALIVE bit in shmem */
2872                 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2873                 bnx2x_drv_pulse(bp);
2874                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2875                 bnx2x_save_statistics(bp);
2876         }
2877
2878         /* wait till consumers catch up with producers in all queues */
2879         bnx2x_drain_tx_queues(bp);
2880
2881         /* if VF indicate to PF this function is going down (PF will delete sp
2882          * elements and clear initializations
2883          */
2884         if (IS_VF(bp))
2885                 bnx2x_vfpf_close_vf(bp);
2886         else if (unload_mode != UNLOAD_RECOVERY)
2887                 /* if this is a normal/close unload need to clean up chip*/
2888                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2889         else {
2890                 /* Send the UNLOAD_REQUEST to the MCP */
2891                 bnx2x_send_unload_req(bp, unload_mode);
2892
2893                 /*
2894                  * Prevent transactions to host from the functions on the
2895                  * engine that doesn't reset global blocks in case of global
2896                  * attention once gloabl blocks are reset and gates are opened
2897                  * (the engine which leader will perform the recovery
2898                  * last).
2899                  */
2900                 if (!CHIP_IS_E1x(bp))
2901                         bnx2x_pf_disable(bp);
2902
2903                 /* Disable HW interrupts, NAPI */
2904                 bnx2x_netif_stop(bp, 1);
2905                 /* Delete all NAPI objects */
2906                 bnx2x_del_all_napi(bp);
2907                 if (CNIC_LOADED(bp))
2908                         bnx2x_del_all_napi_cnic(bp);
2909                 /* Release IRQs */
2910                 bnx2x_free_irq(bp);
2911
2912                 /* Report UNLOAD_DONE to MCP */
2913                 bnx2x_send_unload_done(bp, false);
2914         }
2915
2916         /*
2917          * At this stage no more interrupts will arrive so we may safly clean
2918          * the queueable objects here in case they failed to get cleaned so far.
2919          */
2920         if (IS_PF(bp))
2921                 bnx2x_squeeze_objects(bp);
2922
2923         /* There should be no more pending SP commands at this stage */
2924         bp->sp_state = 0;
2925
2926         bp->port.pmf = 0;
2927
2928         /* Free SKBs, SGEs, TPA pool and driver internals */
2929         bnx2x_free_skbs(bp);
2930         if (CNIC_LOADED(bp))
2931                 bnx2x_free_skbs_cnic(bp);
2932         for_each_rx_queue(bp, i)
2933                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2934
2935         bnx2x_free_fp_mem(bp);
2936         if (CNIC_LOADED(bp))
2937                 bnx2x_free_fp_mem_cnic(bp);
2938
2939         if (IS_PF(bp)) {
2940                 if (CNIC_LOADED(bp))
2941                         bnx2x_free_mem_cnic(bp);
2942                 bnx2x_free_mem(bp);
2943         }
2944         bp->state = BNX2X_STATE_CLOSED;
2945         bp->cnic_loaded = false;
2946
2947         /* Check if there are pending parity attentions. If there are - set
2948          * RECOVERY_IN_PROGRESS.
2949          */
2950         if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
2951                 bnx2x_set_reset_in_progress(bp);
2952
2953                 /* Set RESET_IS_GLOBAL if needed */
2954                 if (global)
2955                         bnx2x_set_reset_global(bp);
2956         }
2957
2958
2959         /* The last driver must disable a "close the gate" if there is no
2960          * parity attention or "process kill" pending.
2961          */
2962         if (IS_PF(bp) &&
2963             !bnx2x_clear_pf_load(bp) &&
2964             bnx2x_reset_is_done(bp, BP_PATH(bp)))
2965                 bnx2x_disable_close_the_gate(bp);
2966
2967         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
2968
2969         return 0;
2970 }
2971
2972 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2973 {
2974         u16 pmcsr;
2975
2976         /* If there is no power capability, silently succeed */
2977         if (!bp->pm_cap) {
2978                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2979                 return 0;
2980         }
2981
2982         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2983
2984         switch (state) {
2985         case PCI_D0:
2986                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2987                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2988                                        PCI_PM_CTRL_PME_STATUS));
2989
2990                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2991                         /* delay required during transition out of D3hot */
2992                         msleep(20);
2993                 break;
2994
2995         case PCI_D3hot:
2996                 /* If there are other clients above don't
2997                    shut down the power */
2998                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2999                         return 0;
3000                 /* Don't shut down the power for emulation and FPGA */
3001                 if (CHIP_REV_IS_SLOW(bp))
3002                         return 0;
3003
3004                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3005                 pmcsr |= 3;
3006
3007                 if (bp->wol)
3008                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3009
3010                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
3011                                       pmcsr);
3012
3013                 /* No more memory access after this point until
3014                 * device is brought back to D0.
3015                 */
3016                 break;
3017
3018         default:
3019                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3020                 return -EINVAL;
3021         }
3022         return 0;
3023 }
3024
3025 /*
3026  * net_device service functions
3027  */
3028 int bnx2x_poll(struct napi_struct *napi, int budget)
3029 {
3030         int work_done = 0;
3031         u8 cos;
3032         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3033                                                  napi);
3034         struct bnx2x *bp = fp->bp;
3035
3036         while (1) {
3037 #ifdef BNX2X_STOP_ON_ERROR
3038                 if (unlikely(bp->panic)) {
3039                         napi_complete(napi);
3040                         return 0;
3041                 }
3042 #endif
3043
3044                 for_each_cos_in_tx_queue(fp, cos)
3045                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3046                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3047
3048                 if (bnx2x_has_rx_work(fp)) {
3049                         work_done += bnx2x_rx_int(fp, budget - work_done);
3050
3051                         /* must not complete if we consumed full budget */
3052                         if (work_done >= budget)
3053                                 break;
3054                 }
3055
3056                 /* Fall out from the NAPI loop if needed */
3057                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3058
3059                         /* No need to update SB for FCoE L2 ring as long as
3060                          * it's connected to the default SB and the SB
3061                          * has been updated when NAPI was scheduled.
3062                          */
3063                         if (IS_FCOE_FP(fp)) {
3064                                 napi_complete(napi);
3065                                 break;
3066                         }
3067                         bnx2x_update_fpsb_idx(fp);
3068                         /* bnx2x_has_rx_work() reads the status block,
3069                          * thus we need to ensure that status block indices
3070                          * have been actually read (bnx2x_update_fpsb_idx)
3071                          * prior to this check (bnx2x_has_rx_work) so that
3072                          * we won't write the "newer" value of the status block
3073                          * to IGU (if there was a DMA right after
3074                          * bnx2x_has_rx_work and if there is no rmb, the memory
3075                          * reading (bnx2x_update_fpsb_idx) may be postponed
3076                          * to right before bnx2x_ack_sb). In this case there
3077                          * will never be another interrupt until there is
3078                          * another update of the status block, while there
3079                          * is still unhandled work.
3080                          */
3081                         rmb();
3082
3083                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3084                                 napi_complete(napi);
3085                                 /* Re-enable interrupts */
3086                                 DP(NETIF_MSG_RX_STATUS,
3087                                    "Update index to %d\n", fp->fp_hc_idx);
3088                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3089                                              le16_to_cpu(fp->fp_hc_idx),
3090                                              IGU_INT_ENABLE, 1);
3091                                 break;
3092                         }
3093                 }
3094         }
3095
3096         return work_done;
3097 }
3098
3099 /* we split the first BD into headers and data BDs
3100  * to ease the pain of our fellow microcode engineers
3101  * we use one mapping for both BDs
3102  */
3103 static u16 bnx2x_tx_split(struct bnx2x *bp,
3104                           struct bnx2x_fp_txdata *txdata,
3105                           struct sw_tx_bd *tx_buf,
3106                           struct eth_tx_start_bd **tx_bd, u16 hlen,
3107                           u16 bd_prod)
3108 {
3109         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3110         struct eth_tx_bd *d_tx_bd;
3111         dma_addr_t mapping;
3112         int old_len = le16_to_cpu(h_tx_bd->nbytes);
3113
3114         /* first fix first BD */
3115         h_tx_bd->nbytes = cpu_to_le16(hlen);
3116
3117         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3118            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3119
3120         /* now get a new data BD
3121          * (after the pbd) and fill it */
3122         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3123         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3124
3125         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3126                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3127
3128         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3129         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3130         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3131
3132         /* this marks the BD as one that has no individual mapping */
3133         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3134
3135         DP(NETIF_MSG_TX_QUEUED,
3136            "TSO split data size is %d (%x:%x)\n",
3137            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3138
3139         /* update tx_bd */
3140         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3141
3142         return bd_prod;
3143 }
3144
3145 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3146 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3147 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3148 {
3149         __sum16 tsum = (__force __sum16) csum;
3150
3151         if (fix > 0)
3152                 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3153                                   csum_partial(t_header - fix, fix, 0)));
3154
3155         else if (fix < 0)
3156                 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3157                                   csum_partial(t_header, -fix, 0)));
3158
3159         return bswab16(tsum);
3160 }
3161
3162 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3163 {
3164         u32 rc;
3165         __u8 prot = 0;
3166         __be16 protocol;
3167
3168         if (skb->ip_summed != CHECKSUM_PARTIAL)
3169                 return XMIT_PLAIN;
3170
3171         protocol = vlan_get_protocol(skb);
3172         if (protocol == htons(ETH_P_IPV6)) {
3173                 rc = XMIT_CSUM_V6;
3174                 prot = ipv6_hdr(skb)->nexthdr;
3175         } else {
3176                 rc = XMIT_CSUM_V4;
3177                 prot = ip_hdr(skb)->protocol;
3178         }
3179
3180         if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3181                 if (inner_ip_hdr(skb)->version == 6) {
3182                         rc |= XMIT_CSUM_ENC_V6;
3183                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3184                                 rc |= XMIT_CSUM_TCP;
3185                 } else {
3186                         rc |= XMIT_CSUM_ENC_V4;
3187                         if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3188                                 rc |= XMIT_CSUM_TCP;
3189                 }
3190         }
3191         if (prot == IPPROTO_TCP)
3192                 rc |= XMIT_CSUM_TCP;
3193
3194         if (skb_is_gso_v6(skb)) {
3195                 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6);
3196                 if (rc & XMIT_CSUM_ENC)
3197                         rc |= XMIT_GSO_ENC_V6;
3198         } else if (skb_is_gso(skb)) {
3199                 rc |= (XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP);
3200                 if (rc & XMIT_CSUM_ENC)
3201                         rc |= XMIT_GSO_ENC_V4;
3202         }
3203
3204         return rc;
3205 }
3206
3207 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3208 /* check if packet requires linearization (packet is too fragmented)
3209    no need to check fragmentation if page size > 8K (there will be no
3210    violation to FW restrictions) */
3211 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3212                              u32 xmit_type)
3213 {
3214         int to_copy = 0;
3215         int hlen = 0;
3216         int first_bd_sz = 0;
3217
3218         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3219         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3220
3221                 if (xmit_type & XMIT_GSO) {
3222                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3223                         /* Check if LSO packet needs to be copied:
3224                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3225                         int wnd_size = MAX_FETCH_BD - 3;
3226                         /* Number of windows to check */
3227                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3228                         int wnd_idx = 0;
3229                         int frag_idx = 0;
3230                         u32 wnd_sum = 0;
3231
3232                         /* Headers length */
3233                         hlen = (int)(skb_transport_header(skb) - skb->data) +
3234                                 tcp_hdrlen(skb);
3235
3236                         /* Amount of data (w/o headers) on linear part of SKB*/
3237                         first_bd_sz = skb_headlen(skb) - hlen;
3238
3239                         wnd_sum  = first_bd_sz;
3240
3241                         /* Calculate the first sum - it's special */
3242                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3243                                 wnd_sum +=
3244                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3245
3246                         /* If there was data on linear skb data - check it */
3247                         if (first_bd_sz > 0) {
3248                                 if (unlikely(wnd_sum < lso_mss)) {
3249                                         to_copy = 1;
3250                                         goto exit_lbl;
3251                                 }
3252
3253                                 wnd_sum -= first_bd_sz;
3254                         }
3255
3256                         /* Others are easier: run through the frag list and
3257                            check all windows */
3258                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3259                                 wnd_sum +=
3260                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3261
3262                                 if (unlikely(wnd_sum < lso_mss)) {
3263                                         to_copy = 1;
3264                                         break;
3265                                 }
3266                                 wnd_sum -=
3267                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3268                         }
3269                 } else {
3270                         /* in non-LSO too fragmented packet should always
3271                            be linearized */
3272                         to_copy = 1;
3273                 }
3274         }
3275
3276 exit_lbl:
3277         if (unlikely(to_copy))
3278                 DP(NETIF_MSG_TX_QUEUED,
3279                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3280                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3281                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3282
3283         return to_copy;
3284 }
3285 #endif
3286
3287 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3288                                  u32 xmit_type)
3289 {
3290         struct ipv6hdr *ipv6;
3291
3292         *parsing_data |= (skb_shinfo(skb)->gso_size <<
3293                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3294                               ETH_TX_PARSE_BD_E2_LSO_MSS;
3295
3296         if (xmit_type & XMIT_GSO_ENC_V6)
3297                 ipv6 = inner_ipv6_hdr(skb);
3298         else if (xmit_type & XMIT_GSO_V6)
3299                 ipv6 = ipv6_hdr(skb);
3300         else
3301                 ipv6 = NULL;
3302
3303         if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3304                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3305 }
3306
3307 /**
3308  * bnx2x_set_pbd_gso - update PBD in GSO case.
3309  *
3310  * @skb:        packet skb
3311  * @pbd:        parse BD
3312  * @xmit_type:  xmit flags
3313  */
3314 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3315                               struct eth_tx_parse_bd_e1x *pbd,
3316                               struct eth_tx_start_bd *tx_start_bd,
3317                               u32 xmit_type)
3318 {
3319         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3320         pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3321         pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3322
3323         if (xmit_type & XMIT_GSO_V4) {
3324                 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3325                 pbd->tcp_pseudo_csum =
3326                         bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3327                                                    ip_hdr(skb)->daddr,
3328                                                    0, IPPROTO_TCP, 0));
3329
3330                 /* GSO on 57710/57711 needs FW to calculate IP checksum */
3331                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3332         } else {
3333                 pbd->tcp_pseudo_csum =
3334                         bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3335                                                  &ipv6_hdr(skb)->daddr,
3336                                                  0, IPPROTO_TCP, 0));
3337         }
3338
3339         pbd->global_data |=
3340                 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3341 }
3342
3343 /**
3344  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3345  *
3346  * @bp:                 driver handle
3347  * @skb:                packet skb
3348  * @parsing_data:       data to be updated
3349  * @xmit_type:          xmit flags
3350  *
3351  * 57712/578xx related, when skb has encapsulation
3352  */
3353 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3354                                  u32 *parsing_data, u32 xmit_type)
3355 {
3356         *parsing_data |=
3357                 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3358                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3359                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3360
3361         if (xmit_type & XMIT_CSUM_TCP) {
3362                 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3363                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3364                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3365
3366                 return skb_inner_transport_header(skb) +
3367                         inner_tcp_hdrlen(skb) - skb->data;
3368         }
3369
3370         /* We support checksum offload for TCP and UDP only.
3371          * No need to pass the UDP header length - it's a constant.
3372          */
3373         return skb_inner_transport_header(skb) +
3374                 sizeof(struct udphdr) - skb->data;
3375 }
3376
3377 /**
3378  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3379  *
3380  * @bp:                 driver handle
3381  * @skb:                packet skb
3382  * @parsing_data:       data to be updated
3383  * @xmit_type:          xmit flags
3384  *
3385  * 57712/578xx related
3386  */
3387 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3388                                 u32 *parsing_data, u32 xmit_type)
3389 {
3390         *parsing_data |=
3391                 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3392                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3393                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3394
3395         if (xmit_type & XMIT_CSUM_TCP) {
3396                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3397                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3398                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3399
3400                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3401         }
3402         /* We support checksum offload for TCP and UDP only.
3403          * No need to pass the UDP header length - it's a constant.
3404          */
3405         return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3406 }
3407
3408 /* set FW indication according to inner or outer protocols if tunneled */
3409 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3410                                struct eth_tx_start_bd *tx_start_bd,
3411                                u32 xmit_type)
3412 {
3413         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3414
3415         if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3416                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3417
3418         if (!(xmit_type & XMIT_CSUM_TCP))
3419                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3420 }
3421
3422 /**
3423  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3424  *
3425  * @bp:         driver handle
3426  * @skb:        packet skb
3427  * @pbd:        parse BD to be updated
3428  * @xmit_type:  xmit flags
3429  */
3430 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3431                              struct eth_tx_parse_bd_e1x *pbd,
3432                              u32 xmit_type)
3433 {
3434         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3435
3436         /* for now NS flag is not used in Linux */
3437         pbd->global_data =
3438                 cpu_to_le16(hlen |
3439                             ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3440                              ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3441
3442         pbd->ip_hlen_w = (skb_transport_header(skb) -
3443                         skb_network_header(skb)) >> 1;
3444
3445         hlen += pbd->ip_hlen_w;
3446
3447         /* We support checksum offload for TCP and UDP only */
3448         if (xmit_type & XMIT_CSUM_TCP)
3449                 hlen += tcp_hdrlen(skb) / 2;
3450         else
3451                 hlen += sizeof(struct udphdr) / 2;
3452
3453         pbd->total_hlen_w = cpu_to_le16(hlen);
3454         hlen = hlen*2;
3455
3456         if (xmit_type & XMIT_CSUM_TCP) {
3457                 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3458
3459         } else {
3460                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3461
3462                 DP(NETIF_MSG_TX_QUEUED,
3463                    "hlen %d  fix %d  csum before fix %x\n",
3464                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3465
3466                 /* HW bug: fixup the CSUM */
3467                 pbd->tcp_pseudo_csum =
3468                         bnx2x_csum_fix(skb_transport_header(skb),
3469                                        SKB_CS(skb), fix);
3470
3471                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3472                    pbd->tcp_pseudo_csum);
3473         }
3474
3475         return hlen;
3476 }
3477
3478 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3479                                       struct eth_tx_parse_bd_e2 *pbd_e2,
3480                                       struct eth_tx_parse_2nd_bd *pbd2,
3481                                       u16 *global_data,
3482                                       u32 xmit_type)
3483 {
3484         u16 hlen_w = 0;
3485         u8 outerip_off, outerip_len = 0;
3486         /* from outer IP to transport */
3487         hlen_w = (skb_inner_transport_header(skb) -
3488                   skb_network_header(skb)) >> 1;
3489
3490         /* transport len */
3491         if (xmit_type & XMIT_CSUM_TCP)
3492                 hlen_w += inner_tcp_hdrlen(skb) >> 1;
3493         else
3494                 hlen_w += sizeof(struct udphdr) >> 1;
3495
3496         pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3497
3498         if (xmit_type & XMIT_CSUM_ENC_V4) {
3499                 struct iphdr *iph = ip_hdr(skb);
3500                 pbd2->fw_ip_csum_wo_len_flags_frag =
3501                         bswab16(csum_fold((~iph->check) -
3502                                           iph->tot_len - iph->frag_off));
3503         } else {
3504                 pbd2->fw_ip_hdr_to_payload_w =
3505                         hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3506         }
3507
3508         pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3509
3510         pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3511
3512         if (xmit_type & XMIT_GSO_V4) {
3513                 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3514
3515                 pbd_e2->data.tunnel_data.pseudo_csum =
3516                         bswab16(~csum_tcpudp_magic(
3517                                         inner_ip_hdr(skb)->saddr,
3518                                         inner_ip_hdr(skb)->daddr,
3519                                         0, IPPROTO_TCP, 0));
3520
3521                 outerip_len = ip_hdr(skb)->ihl << 1;
3522         } else {
3523                 pbd_e2->data.tunnel_data.pseudo_csum =
3524                         bswab16(~csum_ipv6_magic(
3525                                         &inner_ipv6_hdr(skb)->saddr,
3526                                         &inner_ipv6_hdr(skb)->daddr,
3527                                         0, IPPROTO_TCP, 0));
3528         }
3529
3530         outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3531
3532         *global_data |=
3533                 outerip_off |
3534                 (!!(xmit_type & XMIT_CSUM_V6) <<
3535                         ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3536                 (outerip_len <<
3537                         ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3538                 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3539                         ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3540
3541         if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3542                 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3543                 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3544         }
3545 }
3546
3547 /* called with netif_tx_lock
3548  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3549  * netif_wake_queue()
3550  */
3551 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3552 {
3553         struct bnx2x *bp = netdev_priv(dev);
3554
3555         struct netdev_queue *txq;
3556         struct bnx2x_fp_txdata *txdata;
3557         struct sw_tx_bd *tx_buf;
3558         struct eth_tx_start_bd *tx_start_bd, *first_bd;
3559         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3560         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3561         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3562         struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3563         u32 pbd_e2_parsing_data = 0;
3564         u16 pkt_prod, bd_prod;
3565         int nbd, txq_index;
3566         dma_addr_t mapping;
3567         u32 xmit_type = bnx2x_xmit_type(bp, skb);
3568         int i;
3569         u8 hlen = 0;
3570         __le16 pkt_size = 0;
3571         struct ethhdr *eth;
3572         u8 mac_type = UNICAST_ADDRESS;
3573
3574 #ifdef BNX2X_STOP_ON_ERROR
3575         if (unlikely(bp->panic))
3576                 return NETDEV_TX_BUSY;
3577 #endif
3578
3579         txq_index = skb_get_queue_mapping(skb);
3580         txq = netdev_get_tx_queue(dev, txq_index);
3581
3582         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3583
3584         txdata = &bp->bnx2x_txq[txq_index];
3585
3586         /* enable this debug print to view the transmission queue being used
3587         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3588            txq_index, fp_index, txdata_index); */
3589
3590         /* enable this debug print to view the tranmission details
3591         DP(NETIF_MSG_TX_QUEUED,
3592            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3593            txdata->cid, fp_index, txdata_index, txdata, fp); */
3594
3595         if (unlikely(bnx2x_tx_avail(bp, txdata) <
3596                         skb_shinfo(skb)->nr_frags +
3597                         BDS_PER_TX_PKT +
3598                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3599                 /* Handle special storage cases separately */
3600                 if (txdata->tx_ring_size == 0) {
3601                         struct bnx2x_eth_q_stats *q_stats =
3602                                 bnx2x_fp_qstats(bp, txdata->parent_fp);
3603                         q_stats->driver_filtered_tx_pkt++;
3604                         dev_kfree_skb(skb);
3605                         return NETDEV_TX_OK;
3606                 }
3607                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3608                 netif_tx_stop_queue(txq);
3609                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3610
3611                 return NETDEV_TX_BUSY;
3612         }
3613
3614         DP(NETIF_MSG_TX_QUEUED,
3615            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3616            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3617            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3618            skb->len);
3619
3620         eth = (struct ethhdr *)skb->data;
3621
3622         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3623         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3624                 if (is_broadcast_ether_addr(eth->h_dest))
3625                         mac_type = BROADCAST_ADDRESS;
3626                 else
3627                         mac_type = MULTICAST_ADDRESS;
3628         }
3629
3630 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3631         /* First, check if we need to linearize the skb (due to FW
3632            restrictions). No need to check fragmentation if page size > 8K
3633            (there will be no violation to FW restrictions) */
3634         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3635                 /* Statistics of linearization */
3636                 bp->lin_cnt++;
3637                 if (skb_linearize(skb) != 0) {
3638                         DP(NETIF_MSG_TX_QUEUED,
3639                            "SKB linearization failed - silently dropping this SKB\n");
3640                         dev_kfree_skb_any(skb);
3641                         return NETDEV_TX_OK;
3642                 }
3643         }
3644 #endif
3645         /* Map skb linear data for DMA */
3646         mapping = dma_map_single(&bp->pdev->dev, skb->data,
3647                                  skb_headlen(skb), DMA_TO_DEVICE);
3648         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3649                 DP(NETIF_MSG_TX_QUEUED,
3650                    "SKB mapping failed - silently dropping this SKB\n");
3651                 dev_kfree_skb_any(skb);
3652                 return NETDEV_TX_OK;
3653         }
3654         /*
3655         Please read carefully. First we use one BD which we mark as start,
3656         then we have a parsing info BD (used for TSO or xsum),
3657         and only then we have the rest of the TSO BDs.
3658         (don't forget to mark the last one as last,
3659         and to unmap only AFTER you write to the BD ...)
3660         And above all, all pdb sizes are in words - NOT DWORDS!
3661         */
3662
3663         /* get current pkt produced now - advance it just before sending packet
3664          * since mapping of pages may fail and cause packet to be dropped
3665          */
3666         pkt_prod = txdata->tx_pkt_prod;
3667         bd_prod = TX_BD(txdata->tx_bd_prod);
3668
3669         /* get a tx_buf and first BD
3670          * tx_start_bd may be changed during SPLIT,
3671          * but first_bd will always stay first
3672          */
3673         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3674         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3675         first_bd = tx_start_bd;
3676
3677         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3678
3679         /* header nbd: indirectly zero other flags! */
3680         tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3681
3682         /* remember the first BD of the packet */
3683         tx_buf->first_bd = txdata->tx_bd_prod;
3684         tx_buf->skb = skb;
3685         tx_buf->flags = 0;
3686
3687         DP(NETIF_MSG_TX_QUEUED,
3688            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3689            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3690
3691         if (vlan_tx_tag_present(skb)) {
3692                 tx_start_bd->vlan_or_ethertype =
3693                     cpu_to_le16(vlan_tx_tag_get(skb));
3694                 tx_start_bd->bd_flags.as_bitfield |=
3695                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3696         } else {
3697                 /* when transmitting in a vf, start bd must hold the ethertype
3698                  * for fw to enforce it
3699                  */
3700                 if (IS_VF(bp))
3701                         tx_start_bd->vlan_or_ethertype =
3702                                 cpu_to_le16(ntohs(eth->h_proto));
3703                 else
3704                         /* used by FW for packet accounting */
3705                         tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3706         }
3707
3708         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3709
3710         /* turn on parsing and get a BD */
3711         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3712
3713         if (xmit_type & XMIT_CSUM)
3714                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3715
3716         if (!CHIP_IS_E1x(bp)) {
3717                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3718                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3719
3720                 if (xmit_type & XMIT_CSUM_ENC) {
3721                         u16 global_data = 0;
3722
3723                         /* Set PBD in enc checksum offload case */
3724                         hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3725                                                       &pbd_e2_parsing_data,
3726                                                       xmit_type);
3727
3728                         /* turn on 2nd parsing and get a BD */
3729                         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3730
3731                         pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3732
3733                         memset(pbd2, 0, sizeof(*pbd2));
3734
3735                         pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3736                                 (skb_inner_network_header(skb) -
3737                                  skb->data) >> 1;
3738
3739                         if (xmit_type & XMIT_GSO_ENC)
3740                                 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3741                                                           &global_data,
3742                                                           xmit_type);
3743
3744                         pbd2->global_data = cpu_to_le16(global_data);
3745
3746                         /* add addition parse BD indication to start BD */
3747                         SET_FLAG(tx_start_bd->general_data,
3748                                  ETH_TX_START_BD_PARSE_NBDS, 1);
3749                         /* set encapsulation flag in start BD */
3750                         SET_FLAG(tx_start_bd->general_data,
3751                                  ETH_TX_START_BD_TUNNEL_EXIST, 1);
3752                         nbd++;
3753                 } else if (xmit_type & XMIT_CSUM) {
3754                         /* Set PBD in checksum offload case w/o encapsulation */
3755                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3756                                                      &pbd_e2_parsing_data,
3757                                                      xmit_type);
3758                 }
3759
3760                 /* Add the macs to the parsing BD this is a vf */
3761                 if (IS_VF(bp)) {
3762                         /* override GRE parameters in BD */
3763                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3764                                               &pbd_e2->data.mac_addr.src_mid,
3765                                               &pbd_e2->data.mac_addr.src_lo,
3766                                               eth->h_source);
3767
3768                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3769                                               &pbd_e2->data.mac_addr.dst_mid,
3770                                               &pbd_e2->data.mac_addr.dst_lo,
3771                                               eth->h_dest);
3772                 }
3773
3774                 SET_FLAG(pbd_e2_parsing_data,
3775                          ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3776         } else {
3777                 u16 global_data = 0;
3778                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3779                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3780                 /* Set PBD in checksum offload case */
3781                 if (xmit_type & XMIT_CSUM)
3782                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3783
3784                 SET_FLAG(global_data,
3785                          ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3786                 pbd_e1x->global_data |= cpu_to_le16(global_data);
3787         }
3788
3789         /* Setup the data pointer of the first BD of the packet */
3790         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3791         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3792         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3793         pkt_size = tx_start_bd->nbytes;
3794
3795         DP(NETIF_MSG_TX_QUEUED,
3796            "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
3797            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3798            le16_to_cpu(tx_start_bd->nbytes),
3799            tx_start_bd->bd_flags.as_bitfield,
3800            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3801
3802         if (xmit_type & XMIT_GSO) {
3803
3804                 DP(NETIF_MSG_TX_QUEUED,
3805                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3806                    skb->len, hlen, skb_headlen(skb),
3807                    skb_shinfo(skb)->gso_size);
3808
3809                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3810
3811                 if (unlikely(skb_headlen(skb) > hlen)) {
3812                         nbd++;
3813                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3814                                                  &tx_start_bd, hlen,
3815                                                  bd_prod);
3816                 }
3817                 if (!CHIP_IS_E1x(bp))
3818                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3819                                              xmit_type);
3820                 else
3821                         bnx2x_set_pbd_gso(skb, pbd_e1x, tx_start_bd,
3822                                           xmit_type);
3823         }
3824
3825         /* Set the PBD's parsing_data field if not zero
3826          * (for the chips newer than 57711).
3827          */
3828         if (pbd_e2_parsing_data)
3829                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3830
3831         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3832
3833         /* Handle fragmented skb */
3834         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3835                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3836
3837                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3838                                            skb_frag_size(frag), DMA_TO_DEVICE);
3839                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3840                         unsigned int pkts_compl = 0, bytes_compl = 0;
3841
3842                         DP(NETIF_MSG_TX_QUEUED,
3843                            "Unable to map page - dropping packet...\n");
3844
3845                         /* we need unmap all buffers already mapped
3846                          * for this SKB;
3847                          * first_bd->nbd need to be properly updated
3848                          * before call to bnx2x_free_tx_pkt
3849                          */
3850                         first_bd->nbd = cpu_to_le16(nbd);
3851                         bnx2x_free_tx_pkt(bp, txdata,
3852                                           TX_BD(txdata->tx_pkt_prod),
3853                                           &pkts_compl, &bytes_compl);
3854                         return NETDEV_TX_OK;
3855                 }
3856
3857                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3858                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3859                 if (total_pkt_bd == NULL)
3860                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3861
3862                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3863                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3864                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3865                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3866                 nbd++;
3867
3868                 DP(NETIF_MSG_TX_QUEUED,
3869                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
3870                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3871                    le16_to_cpu(tx_data_bd->nbytes));
3872         }
3873
3874         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3875
3876         /* update with actual num BDs */
3877         first_bd->nbd = cpu_to_le16(nbd);
3878
3879         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3880
3881         /* now send a tx doorbell, counting the next BD
3882          * if the packet contains or ends with it
3883          */
3884         if (TX_BD_POFF(bd_prod) < nbd)
3885                 nbd++;
3886
3887         /* total_pkt_bytes should be set on the first data BD if
3888          * it's not an LSO packet and there is more than one
3889          * data BD. In this case pkt_size is limited by an MTU value.
3890          * However we prefer to set it for an LSO packet (while we don't
3891          * have to) in order to save some CPU cycles in a none-LSO
3892          * case, when we much more care about them.
3893          */
3894         if (total_pkt_bd != NULL)
3895                 total_pkt_bd->total_pkt_bytes = pkt_size;
3896
3897         if (pbd_e1x)
3898                 DP(NETIF_MSG_TX_QUEUED,
3899                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
3900                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3901                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3902                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3903                     le16_to_cpu(pbd_e1x->total_hlen_w));
3904         if (pbd_e2)
3905                 DP(NETIF_MSG_TX_QUEUED,
3906                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
3907                    pbd_e2,
3908                    pbd_e2->data.mac_addr.dst_hi,
3909                    pbd_e2->data.mac_addr.dst_mid,
3910                    pbd_e2->data.mac_addr.dst_lo,
3911                    pbd_e2->data.mac_addr.src_hi,
3912                    pbd_e2->data.mac_addr.src_mid,
3913                    pbd_e2->data.mac_addr.src_lo,
3914                    pbd_e2->parsing_data);
3915         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
3916
3917         netdev_tx_sent_queue(txq, skb->len);
3918
3919         skb_tx_timestamp(skb);
3920
3921         txdata->tx_pkt_prod++;
3922         /*
3923          * Make sure that the BD data is updated before updating the producer
3924          * since FW might read the BD right after the producer is updated.
3925          * This is only applicable for weak-ordered memory model archs such
3926          * as IA-64. The following barrier is also mandatory since FW will
3927          * assumes packets must have BDs.
3928          */
3929         wmb();
3930
3931         txdata->tx_db.data.prod += nbd;
3932         barrier();
3933
3934         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
3935
3936         mmiowb();
3937
3938         txdata->tx_bd_prod += nbd;
3939
3940         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
3941                 netif_tx_stop_queue(txq);
3942
3943                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
3944                  * ordering of set_bit() in netif_tx_stop_queue() and read of
3945                  * fp->bd_tx_cons */
3946                 smp_mb();
3947
3948                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3949                 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
3950                         netif_tx_wake_queue(txq);
3951         }
3952         txdata->tx_pkt++;
3953
3954         return NETDEV_TX_OK;
3955 }
3956
3957 /**
3958  * bnx2x_setup_tc - routine to configure net_device for multi tc
3959  *
3960  * @netdev: net device to configure
3961  * @tc: number of traffic classes to enable
3962  *
3963  * callback connected to the ndo_setup_tc function pointer
3964  */
3965 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
3966 {
3967         int cos, prio, count, offset;
3968         struct bnx2x *bp = netdev_priv(dev);
3969
3970         /* setup tc must be called under rtnl lock */
3971         ASSERT_RTNL();
3972
3973         /* no traffic classes requested. aborting */
3974         if (!num_tc) {
3975                 netdev_reset_tc(dev);
3976                 return 0;
3977         }
3978
3979         /* requested to support too many traffic classes */
3980         if (num_tc > bp->max_cos) {
3981                 BNX2X_ERR("support for too many traffic classes requested: %d. max supported is %d\n",
3982                           num_tc, bp->max_cos);
3983                 return -EINVAL;
3984         }
3985
3986         /* declare amount of supported traffic classes */
3987         if (netdev_set_num_tc(dev, num_tc)) {
3988                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
3989                 return -EINVAL;
3990         }
3991
3992         /* configure priority to traffic class mapping */
3993         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
3994                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
3995                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3996                    "mapping priority %d to tc %d\n",
3997                    prio, bp->prio_to_cos[prio]);
3998         }
3999
4000
4001         /* Use this configuration to diffrentiate tc0 from other COSes
4002            This can be used for ets or pfc, and save the effort of setting
4003            up a multio class queue disc or negotiating DCBX with a switch
4004         netdev_set_prio_tc_map(dev, 0, 0);
4005         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4006         for (prio = 1; prio < 16; prio++) {
4007                 netdev_set_prio_tc_map(dev, prio, 1);
4008                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4009         } */
4010
4011         /* configure traffic class to transmission queue mapping */
4012         for (cos = 0; cos < bp->max_cos; cos++) {
4013                 count = BNX2X_NUM_ETH_QUEUES(bp);
4014                 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4015                 netdev_set_tc_queue(dev, cos, count, offset);
4016                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4017                    "mapping tc %d to offset %d count %d\n",
4018                    cos, offset, count);
4019         }
4020
4021         return 0;
4022 }
4023
4024 /* called with rtnl_lock */
4025 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4026 {
4027         struct sockaddr *addr = p;
4028         struct bnx2x *bp = netdev_priv(dev);
4029         int rc = 0;
4030
4031         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4032                 BNX2X_ERR("Requested MAC address is not valid\n");
4033                 return -EINVAL;
4034         }
4035
4036         if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4037             !is_zero_ether_addr(addr->sa_data)) {
4038                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4039                 return -EINVAL;
4040         }
4041
4042         if (netif_running(dev))  {
4043                 rc = bnx2x_set_eth_mac(bp, false);
4044                 if (rc)
4045                         return rc;
4046         }
4047
4048         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4049
4050         if (netif_running(dev))
4051                 rc = bnx2x_set_eth_mac(bp, true);
4052
4053         return rc;
4054 }
4055
4056 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4057 {
4058         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4059         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4060         u8 cos;
4061
4062         /* Common */
4063
4064         if (IS_FCOE_IDX(fp_index)) {
4065                 memset(sb, 0, sizeof(union host_hc_status_block));
4066                 fp->status_blk_mapping = 0;
4067         } else {
4068                 /* status blocks */
4069                 if (!CHIP_IS_E1x(bp))
4070                         BNX2X_PCI_FREE(sb->e2_sb,
4071                                        bnx2x_fp(bp, fp_index,
4072                                                 status_blk_mapping),
4073                                        sizeof(struct host_hc_status_block_e2));
4074                 else
4075                         BNX2X_PCI_FREE(sb->e1x_sb,
4076                                        bnx2x_fp(bp, fp_index,
4077                                                 status_blk_mapping),
4078                                        sizeof(struct host_hc_status_block_e1x));
4079         }
4080
4081         /* Rx */
4082         if (!skip_rx_queue(bp, fp_index)) {
4083                 bnx2x_free_rx_bds(fp);
4084
4085                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4086                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4087                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4088                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
4089                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
4090
4091                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4092                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
4093                                sizeof(struct eth_fast_path_rx_cqe) *
4094                                NUM_RCQ_BD);
4095
4096                 /* SGE ring */
4097                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4098                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4099                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
4100                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4101         }
4102
4103         /* Tx */
4104         if (!skip_tx_queue(bp, fp_index)) {
4105                 /* fastpath tx rings: tx_buf tx_desc */
4106                 for_each_cos_in_tx_queue(fp, cos) {
4107                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4108
4109                         DP(NETIF_MSG_IFDOWN,
4110                            "freeing tx memory of fp %d cos %d cid %d\n",
4111                            fp_index, cos, txdata->cid);
4112
4113                         BNX2X_FREE(txdata->tx_buf_ring);
4114                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
4115                                 txdata->tx_desc_mapping,
4116                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4117                 }
4118         }
4119         /* end of fastpath */
4120 }
4121
4122 void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4123 {
4124         int i;
4125         for_each_cnic_queue(bp, i)
4126                 bnx2x_free_fp_mem_at(bp, i);
4127 }
4128
4129 void bnx2x_free_fp_mem(struct bnx2x *bp)
4130 {
4131         int i;
4132         for_each_eth_queue(bp, i)
4133                 bnx2x_free_fp_mem_at(bp, i);
4134 }
4135
4136 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4137 {
4138         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4139         if (!CHIP_IS_E1x(bp)) {
4140                 bnx2x_fp(bp, index, sb_index_values) =
4141                         (__le16 *)status_blk.e2_sb->sb.index_values;
4142                 bnx2x_fp(bp, index, sb_running_index) =
4143                         (__le16 *)status_blk.e2_sb->sb.running_index;
4144         } else {
4145                 bnx2x_fp(bp, index, sb_index_values) =
4146                         (__le16 *)status_blk.e1x_sb->sb.index_values;
4147                 bnx2x_fp(bp, index, sb_running_index) =
4148                         (__le16 *)status_blk.e1x_sb->sb.running_index;
4149         }
4150 }
4151
4152 /* Returns the number of actually allocated BDs */
4153 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4154                               int rx_ring_size)
4155 {
4156         struct bnx2x *bp = fp->bp;
4157         u16 ring_prod, cqe_ring_prod;
4158         int i, failure_cnt = 0;
4159
4160         fp->rx_comp_cons = 0;
4161         cqe_ring_prod = ring_prod = 0;
4162
4163         /* This routine is called only during fo init so
4164          * fp->eth_q_stats.rx_skb_alloc_failed = 0
4165          */
4166         for (i = 0; i < rx_ring_size; i++) {
4167                 if (bnx2x_alloc_rx_data(bp, fp, ring_prod) < 0) {
4168                         failure_cnt++;
4169                         continue;
4170                 }
4171                 ring_prod = NEXT_RX_IDX(ring_prod);
4172                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4173                 WARN_ON(ring_prod <= (i - failure_cnt));
4174         }
4175
4176         if (failure_cnt)
4177                 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4178                           i - failure_cnt, fp->index);
4179
4180         fp->rx_bd_prod = ring_prod;
4181         /* Limit the CQE producer by the CQE ring size */
4182         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4183                                cqe_ring_prod);
4184         fp->rx_pkt = fp->rx_calls = 0;
4185
4186         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4187
4188         return i - failure_cnt;
4189 }
4190
4191 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4192 {
4193         int i;
4194
4195         for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4196                 struct eth_rx_cqe_next_page *nextpg;
4197
4198                 nextpg = (struct eth_rx_cqe_next_page *)
4199                         &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4200                 nextpg->addr_hi =
4201                         cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4202                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4203                 nextpg->addr_lo =
4204                         cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4205                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4206         }
4207 }
4208
4209 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4210 {
4211         union host_hc_status_block *sb;
4212         struct bnx2x_fastpath *fp = &bp->fp[index];
4213         int ring_size = 0;
4214         u8 cos;
4215         int rx_ring_size = 0;
4216
4217         if (!bp->rx_ring_size &&
4218             (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4219                 rx_ring_size = MIN_RX_SIZE_NONTPA;
4220                 bp->rx_ring_size = rx_ring_size;
4221         } else if (!bp->rx_ring_size) {
4222                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4223
4224                 if (CHIP_IS_E3(bp)) {
4225                         u32 cfg = SHMEM_RD(bp,
4226                                            dev_info.port_hw_config[BP_PORT(bp)].
4227                                            default_cfg);
4228
4229                         /* Decrease ring size for 1G functions */
4230                         if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4231                             PORT_HW_CFG_NET_SERDES_IF_SGMII)
4232                                 rx_ring_size /= 10;
4233                 }
4234
4235                 /* allocate at least number of buffers required by FW */
4236                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4237                                      MIN_RX_SIZE_TPA, rx_ring_size);
4238
4239                 bp->rx_ring_size = rx_ring_size;
4240         } else /* if rx_ring_size specified - use it */
4241                 rx_ring_size = bp->rx_ring_size;
4242
4243         DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4244
4245         /* Common */
4246         sb = &bnx2x_fp(bp, index, status_blk);
4247
4248         if (!IS_FCOE_IDX(index)) {
4249                 /* status blocks */
4250                 if (!CHIP_IS_E1x(bp))
4251                         BNX2X_PCI_ALLOC(sb->e2_sb,
4252                                 &bnx2x_fp(bp, index, status_blk_mapping),
4253                                 sizeof(struct host_hc_status_block_e2));
4254                 else
4255                         BNX2X_PCI_ALLOC(sb->e1x_sb,
4256                                 &bnx2x_fp(bp, index, status_blk_mapping),
4257                             sizeof(struct host_hc_status_block_e1x));
4258         }
4259
4260         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4261          * set shortcuts for it.
4262          */
4263         if (!IS_FCOE_IDX(index))
4264                 set_sb_shortcuts(bp, index);
4265
4266         /* Tx */
4267         if (!skip_tx_queue(bp, index)) {
4268                 /* fastpath tx rings: tx_buf tx_desc */
4269                 for_each_cos_in_tx_queue(fp, cos) {
4270                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4271
4272                         DP(NETIF_MSG_IFUP,
4273                            "allocating tx memory of fp %d cos %d\n",
4274                            index, cos);
4275
4276                         BNX2X_ALLOC(txdata->tx_buf_ring,
4277                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
4278                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
4279                                 &txdata->tx_desc_mapping,
4280                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4281                 }
4282         }
4283
4284         /* Rx */
4285         if (!skip_rx_queue(bp, index)) {
4286                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4287                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
4288                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
4289                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
4290                                 &bnx2x_fp(bp, index, rx_desc_mapping),
4291                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4292
4293                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
4294                                 &bnx2x_fp(bp, index, rx_comp_mapping),
4295                                 sizeof(struct eth_fast_path_rx_cqe) *
4296                                 NUM_RCQ_BD);
4297
4298                 /* SGE ring */
4299                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
4300                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
4301                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
4302                                 &bnx2x_fp(bp, index, rx_sge_mapping),
4303                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4304                 /* RX BD ring */
4305                 bnx2x_set_next_page_rx_bd(fp);
4306
4307                 /* CQ ring */
4308                 bnx2x_set_next_page_rx_cq(fp);
4309
4310                 /* BDs */
4311                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4312                 if (ring_size < rx_ring_size)
4313                         goto alloc_mem_err;
4314         }
4315
4316         return 0;
4317
4318 /* handles low memory cases */
4319 alloc_mem_err:
4320         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4321                                                 index, ring_size);
4322         /* FW will drop all packets if queue is not big enough,
4323          * In these cases we disable the queue
4324          * Min size is different for OOO, TPA and non-TPA queues
4325          */
4326         if (ring_size < (fp->disable_tpa ?
4327                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4328                         /* release memory allocated for this queue */
4329                         bnx2x_free_fp_mem_at(bp, index);
4330                         return -ENOMEM;
4331         }
4332         return 0;
4333 }
4334
4335 int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4336 {
4337         if (!NO_FCOE(bp))
4338                 /* FCoE */
4339                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4340                         /* we will fail load process instead of mark
4341                          * NO_FCOE_FLAG
4342                          */
4343                         return -ENOMEM;
4344
4345         return 0;
4346 }
4347
4348 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4349 {
4350         int i;
4351
4352         /* 1. Allocate FP for leading - fatal if error
4353          * 2. Allocate RSS - fix number of queues if error
4354          */
4355
4356         /* leading */
4357         if (bnx2x_alloc_fp_mem_at(bp, 0))
4358                 return -ENOMEM;
4359
4360         /* RSS */
4361         for_each_nondefault_eth_queue(bp, i)
4362                 if (bnx2x_alloc_fp_mem_at(bp, i))
4363                         break;
4364
4365         /* handle memory failures */
4366         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4367                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4368
4369                 WARN_ON(delta < 0);
4370                 bnx2x_shrink_eth_fp(bp, delta);
4371                 if (CNIC_SUPPORT(bp))
4372                         /* move non eth FPs next to last eth FP
4373                          * must be done in that order
4374                          * FCOE_IDX < FWD_IDX < OOO_IDX
4375                          */
4376
4377                         /* move FCoE fp even NO_FCOE_FLAG is on */
4378                         bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4379                 bp->num_ethernet_queues -= delta;
4380                 bp->num_queues = bp->num_ethernet_queues +
4381                                  bp->num_cnic_queues;
4382                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4383                           bp->num_queues + delta, bp->num_queues);
4384         }
4385
4386         return 0;
4387 }
4388
4389 void bnx2x_free_mem_bp(struct bnx2x *bp)
4390 {
4391         int i;
4392
4393         for (i = 0; i < bp->fp_array_size; i++)
4394                 kfree(bp->fp[i].tpa_info);
4395         kfree(bp->fp);
4396         kfree(bp->sp_objs);
4397         kfree(bp->fp_stats);
4398         kfree(bp->bnx2x_txq);
4399         kfree(bp->msix_table);
4400         kfree(bp->ilt);
4401 }
4402
4403 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4404 {
4405         struct bnx2x_fastpath *fp;
4406         struct msix_entry *tbl;
4407         struct bnx2x_ilt *ilt;
4408         int msix_table_size = 0;
4409         int fp_array_size, txq_array_size;
4410         int i;
4411
4412         /*
4413          * The biggest MSI-X table we might need is as a maximum number of fast
4414          * path IGU SBs plus default SB (for PF only).
4415          */
4416         msix_table_size = bp->igu_sb_cnt;
4417         if (IS_PF(bp))
4418                 msix_table_size++;
4419         BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4420
4421         /* fp array: RSS plus CNIC related L2 queues */
4422         fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4423         bp->fp_array_size = fp_array_size;
4424         BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4425
4426         fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4427         if (!fp)
4428                 goto alloc_err;
4429         for (i = 0; i < bp->fp_array_size; i++) {
4430                 fp[i].tpa_info =
4431                         kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4432                                 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4433                 if (!(fp[i].tpa_info))
4434                         goto alloc_err;
4435         }
4436
4437         bp->fp = fp;
4438
4439         /* allocate sp objs */
4440         bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4441                               GFP_KERNEL);
4442         if (!bp->sp_objs)
4443                 goto alloc_err;
4444
4445         /* allocate fp_stats */
4446         bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4447                                GFP_KERNEL);
4448         if (!bp->fp_stats)
4449                 goto alloc_err;
4450
4451         /* Allocate memory for the transmission queues array */
4452         txq_array_size =
4453                 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4454         BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4455
4456         bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4457                                 GFP_KERNEL);
4458         if (!bp->bnx2x_txq)
4459                 goto alloc_err;
4460
4461         /* msix table */
4462         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4463         if (!tbl)
4464                 goto alloc_err;
4465         bp->msix_table = tbl;
4466
4467         /* ilt */
4468         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4469         if (!ilt)
4470                 goto alloc_err;
4471         bp->ilt = ilt;
4472
4473         return 0;
4474 alloc_err:
4475         bnx2x_free_mem_bp(bp);
4476         return -ENOMEM;
4477
4478 }
4479
4480 int bnx2x_reload_if_running(struct net_device *dev)
4481 {
4482         struct bnx2x *bp = netdev_priv(dev);
4483
4484         if (unlikely(!netif_running(dev)))
4485                 return 0;
4486
4487         bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4488         return bnx2x_nic_load(bp, LOAD_NORMAL);
4489 }
4490
4491 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4492 {
4493         u32 sel_phy_idx = 0;
4494         if (bp->link_params.num_phys <= 1)
4495                 return INT_PHY;
4496
4497         if (bp->link_vars.link_up) {
4498                 sel_phy_idx = EXT_PHY1;
4499                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4500                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4501                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4502                         sel_phy_idx = EXT_PHY2;
4503         } else {
4504
4505                 switch (bnx2x_phy_selection(&bp->link_params)) {
4506                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4507                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4508                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4509                        sel_phy_idx = EXT_PHY1;
4510                        break;
4511                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4512                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4513                        sel_phy_idx = EXT_PHY2;
4514                        break;
4515                 }
4516         }
4517
4518         return sel_phy_idx;
4519
4520 }
4521 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4522 {
4523         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4524         /*
4525          * The selected activated PHY is always after swapping (in case PHY
4526          * swapping is enabled). So when swapping is enabled, we need to reverse
4527          * the configuration
4528          */
4529
4530         if (bp->link_params.multi_phy_config &
4531             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4532                 if (sel_phy_idx == EXT_PHY1)
4533                         sel_phy_idx = EXT_PHY2;
4534                 else if (sel_phy_idx == EXT_PHY2)
4535                         sel_phy_idx = EXT_PHY1;
4536         }
4537         return LINK_CONFIG_IDX(sel_phy_idx);
4538 }
4539
4540 #ifdef NETDEV_FCOE_WWNN
4541 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4542 {
4543         struct bnx2x *bp = netdev_priv(dev);
4544         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4545
4546         switch (type) {
4547         case NETDEV_FCOE_WWNN:
4548                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4549                                 cp->fcoe_wwn_node_name_lo);
4550                 break;
4551         case NETDEV_FCOE_WWPN:
4552                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4553                                 cp->fcoe_wwn_port_name_lo);
4554                 break;
4555         default:
4556                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4557                 return -EINVAL;
4558         }
4559
4560         return 0;
4561 }
4562 #endif
4563
4564 /* called with rtnl_lock */
4565 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4566 {
4567         struct bnx2x *bp = netdev_priv(dev);
4568
4569         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4570                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4571                 return -EAGAIN;
4572         }
4573
4574         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4575             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4576                 BNX2X_ERR("Can't support requested MTU size\n");
4577                 return -EINVAL;
4578         }
4579
4580         /* This does not race with packet allocation
4581          * because the actual alloc size is
4582          * only updated as part of load
4583          */
4584         dev->mtu = new_mtu;
4585
4586         return bnx2x_reload_if_running(dev);
4587 }
4588
4589 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4590                                      netdev_features_t features)
4591 {
4592         struct bnx2x *bp = netdev_priv(dev);
4593
4594         /* TPA requires Rx CSUM offloading */
4595         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4596                 features &= ~NETIF_F_LRO;
4597                 features &= ~NETIF_F_GRO;
4598         }
4599
4600         return features;
4601 }
4602
4603 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4604 {
4605         struct bnx2x *bp = netdev_priv(dev);
4606         u32 flags = bp->flags;
4607         bool bnx2x_reload = false;
4608
4609         if (features & NETIF_F_LRO)
4610                 flags |= TPA_ENABLE_FLAG;
4611         else
4612                 flags &= ~TPA_ENABLE_FLAG;
4613
4614         if (features & NETIF_F_GRO)
4615                 flags |= GRO_ENABLE_FLAG;
4616         else
4617                 flags &= ~GRO_ENABLE_FLAG;
4618
4619         if (features & NETIF_F_LOOPBACK) {
4620                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4621                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
4622                         bnx2x_reload = true;
4623                 }
4624         } else {
4625                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4626                         bp->link_params.loopback_mode = LOOPBACK_NONE;
4627                         bnx2x_reload = true;
4628                 }
4629         }
4630
4631         if (flags ^ bp->flags) {
4632                 bp->flags = flags;
4633                 bnx2x_reload = true;
4634         }
4635
4636         if (bnx2x_reload) {
4637                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4638                         return bnx2x_reload_if_running(dev);
4639                 /* else: bnx2x_nic_load() will be called at end of recovery */
4640         }
4641
4642         return 0;
4643 }
4644
4645 void bnx2x_tx_timeout(struct net_device *dev)
4646 {
4647         struct bnx2x *bp = netdev_priv(dev);
4648
4649 #ifdef BNX2X_STOP_ON_ERROR
4650         if (!bp->panic)
4651                 bnx2x_panic();
4652 #endif
4653
4654         smp_mb__before_clear_bit();
4655         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4656         smp_mb__after_clear_bit();
4657
4658         /* This allows the netif to be shutdown gracefully before resetting */
4659         schedule_delayed_work(&bp->sp_rtnl_task, 0);
4660 }
4661
4662 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4663 {
4664         struct net_device *dev = pci_get_drvdata(pdev);
4665         struct bnx2x *bp;
4666
4667         if (!dev) {
4668                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4669                 return -ENODEV;
4670         }
4671         bp = netdev_priv(dev);
4672
4673         rtnl_lock();
4674
4675         pci_save_state(pdev);
4676
4677         if (!netif_running(dev)) {
4678                 rtnl_unlock();
4679                 return 0;
4680         }
4681
4682         netif_device_detach(dev);
4683
4684         bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4685
4686         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4687
4688         rtnl_unlock();
4689
4690         return 0;
4691 }
4692
4693 int bnx2x_resume(struct pci_dev *pdev)
4694 {
4695         struct net_device *dev = pci_get_drvdata(pdev);
4696         struct bnx2x *bp;
4697         int rc;
4698
4699         if (!dev) {
4700                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4701                 return -ENODEV;
4702         }
4703         bp = netdev_priv(dev);
4704
4705         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4706                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4707                 return -EAGAIN;
4708         }
4709
4710         rtnl_lock();
4711
4712         pci_restore_state(pdev);
4713
4714         if (!netif_running(dev)) {
4715                 rtnl_unlock();
4716                 return 0;
4717         }
4718
4719         bnx2x_set_power_state(bp, PCI_D0);
4720         netif_device_attach(dev);
4721
4722         rc = bnx2x_nic_load(bp, LOAD_OPEN);
4723
4724         rtnl_unlock();
4725
4726         return rc;
4727 }
4728
4729
4730 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4731                               u32 cid)
4732 {
4733         /* ustorm cxt validation */
4734         cxt->ustorm_ag_context.cdu_usage =
4735                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4736                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4737         /* xcontext validation */
4738         cxt->xstorm_ag_context.cdu_reserved =
4739                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4740                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4741 }
4742
4743 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4744                                     u8 fw_sb_id, u8 sb_index,
4745                                     u8 ticks)
4746 {
4747
4748         u32 addr = BAR_CSTRORM_INTMEM +
4749                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4750         REG_WR8(bp, addr, ticks);
4751         DP(NETIF_MSG_IFUP,
4752            "port %x fw_sb_id %d sb_index %d ticks %d\n",
4753            port, fw_sb_id, sb_index, ticks);
4754 }
4755
4756 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4757                                     u16 fw_sb_id, u8 sb_index,
4758                                     u8 disable)
4759 {
4760         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4761         u32 addr = BAR_CSTRORM_INTMEM +
4762                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4763         u8 flags = REG_RD8(bp, addr);
4764         /* clear and set */
4765         flags &= ~HC_INDEX_DATA_HC_ENABLED;
4766         flags |= enable_flag;
4767         REG_WR8(bp, addr, flags);
4768         DP(NETIF_MSG_IFUP,
4769            "port %x fw_sb_id %d sb_index %d disable %d\n",
4770            port, fw_sb_id, sb_index, disable);
4771 }
4772
4773 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4774                                     u8 sb_index, u8 disable, u16 usec)
4775 {
4776         int port = BP_PORT(bp);
4777         u8 ticks = usec / BNX2X_BTR;
4778
4779         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4780
4781         disable = disable ? 1 : (usec ? 0 : 1);
4782         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4783 }