drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2012 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/ipv6.h>
  25 #include <net/ip6_checksum.h>
  26 #include <linux/prefetch.h>
  27 #include "bnx2x_cmn.h"
  28 #include "bnx2x_init.h"
  29 #include "bnx2x_sp.h"
  30
  31
  32
  33 /**
  34  * bnx2x_move_fp - move content of the fastpath structure.
  35  *
  36  * @bp:         driver handle
  37  * @from:       source FP index
  38  * @to:         destination FP index
  39  *
  40  * Makes sure the contents of the bp->fp[to].napi is kept
  41  * intact. This is done by first copying the napi struct from
  42  * the target to the source, and then mem copying the entire
  43  * source onto the target
  44  */
  45 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  46 {
  47         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  48         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  49
  50         /* Copy the NAPI object as it has been already initialized */
  51         from_fp->napi = to_fp->napi;
  52
  53         /* Move bnx2x_fastpath contents */
  54         memcpy(to_fp, from_fp, sizeof(*to_fp));
  55         to_fp->index = to;
  56 }
  57
  58 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
  59
  60 /* free skb in the packet ring at pos idx
  61  * return idx of last bd freed
  62  */
  63 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
  64                              u16 idx, unsigned int *pkts_compl,
  65                              unsigned int *bytes_compl)
  66 {
  67         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
  68         struct eth_tx_start_bd *tx_start_bd;
  69         struct eth_tx_bd *tx_data_bd;
  70         struct sk_buff *skb = tx_buf->skb;
  71         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
  72         int nbd;
  73
  74         /* prefetch skb end pointer to speedup dev_kfree_skb() */
  75         prefetch(&skb->end);
  76
  77         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
  78            txdata->txq_index, idx, tx_buf, skb);
  79
  80         /* unmap first bd */
  81         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
  82         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
  83                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
  84
  85
  86         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
  87 #ifdef BNX2X_STOP_ON_ERROR
  88         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
  89                 BNX2X_ERR("BAD nbd!\n");
  90                 bnx2x_panic();
  91         }
  92 #endif
  93         new_cons = nbd + tx_buf->first_bd;
  94
  95         /* Get the next bd */
  96         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  97
  98         /* Skip a parse bd... */
  99         --nbd;
 100         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 101
 102         /* ...and the TSO split header bd since they have no mapping */
 103         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 104                 --nbd;
 105                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 106         }
 107
 108         /* now free frags */
 109         while (nbd > 0) {
 110
 111                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 112                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 113                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 114                 if (--nbd)
 115                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 116         }
 117
 118         /* release skb */
 119         WARN_ON(!skb);
 120         if (likely(skb)) {
 121                 (*pkts_compl)++;
 122                 (*bytes_compl) += skb->len;
 123         }
 124
 125         dev_kfree_skb_any(skb);
 126         tx_buf->first_bd = 0;
 127         tx_buf->skb = NULL;
 128
 129         return new_cons;
 130 }
 131
 132 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 133 {
 134         struct netdev_queue *txq;
 135         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 136         unsigned int pkts_compl = 0, bytes_compl = 0;
 137
 138 #ifdef BNX2X_STOP_ON_ERROR
 139         if (unlikely(bp->panic))
 140                 return -1;
 141 #endif
 142
 143         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 144         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 145         sw_cons = txdata->tx_pkt_cons;
 146
 147         while (sw_cons != hw_cons) {
 148                 u16 pkt_cons;
 149
 150                 pkt_cons = TX_BD(sw_cons);
 151
 152                 DP(NETIF_MSG_TX_DONE,
 153                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 154                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 155
 156                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 157                     &pkts_compl, &bytes_compl);
 158
 159                 sw_cons++;
 160         }
 161
 162         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 163
 164         txdata->tx_pkt_cons = sw_cons;
 165         txdata->tx_bd_cons = bd_cons;
 166
 167         /* Need to make the tx_bd_cons update visible to start_xmit()
 168          * before checking for netif_tx_queue_stopped().  Without the
 169          * memory barrier, there is a small possibility that
 170          * start_xmit() will miss it and cause the queue to be stopped
 171          * forever.
 172          * On the other hand we need an rmb() here to ensure the proper
 173          * ordering of bit testing in the following
 174          * netif_tx_queue_stopped(txq) call.
 175          */
 176         smp_mb();
 177
 178         if (unlikely(netif_tx_queue_stopped(txq))) {
 179                 /* Taking tx_lock() is needed to prevent reenabling the queue
 180                  * while it's empty. This could have happen if rx_action() gets
 181                  * suspended in bnx2x_tx_int() after the condition before
 182                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 183                  *
 184                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 185                  * sends some packets consuming the whole queue again->
 186                  * stops the queue
 187                  */
 188
 189                 __netif_tx_lock(txq, smp_processor_id());
 190
 191                 if ((netif_tx_queue_stopped(txq)) &&
 192                     (bp->state == BNX2X_STATE_OPEN) &&
 193                     (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3))
 194                         netif_tx_wake_queue(txq);
 195
 196                 __netif_tx_unlock(txq);
 197         }
 198         return 0;
 199 }
 200
 201 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 202                                              u16 idx)
 203 {
 204         u16 last_max = fp->last_max_sge;
 205
 206         if (SUB_S16(idx, last_max) > 0)
 207                 fp->last_max_sge = idx;
 208 }
 209
 210 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 211                                          u16 sge_len,
 212                                          struct eth_end_agg_rx_cqe *cqe)
 213 {
 214         struct bnx2x *bp = fp->bp;
 215         u16 last_max, last_elem, first_elem;
 216         u16 delta = 0;
 217         u16 i;
 218
 219         if (!sge_len)
 220                 return;
 221
 222         /* First mark all used pages */
 223         for (i = 0; i < sge_len; i++)
 224                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 225                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 226
 227         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 228            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 229
 230         /* Here we assume that the last SGE index is the biggest */
 231         prefetch((void *)(fp->sge_mask));
 232         bnx2x_update_last_max_sge(fp,
 233                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 234
 235         last_max = RX_SGE(fp->last_max_sge);
 236         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 237         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 238
 239         /* If ring is not full */
 240         if (last_elem + 1 != first_elem)
 241                 last_elem++;
 242
 243         /* Now update the prod */
 244         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 245                 if (likely(fp->sge_mask[i]))
 246                         break;
 247
 248                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 249                 delta += BIT_VEC64_ELEM_SZ;
 250         }
 251
 252         if (delta > 0) {
 253                 fp->rx_sge_prod += delta;
 254                 /* clear page-end entries */
 255                 bnx2x_clear_sge_mask_next_elems(fp);
 256         }
 257
 258         DP(NETIF_MSG_RX_STATUS,
 259            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 260            fp->last_max_sge, fp->rx_sge_prod);
 261 }
 262
 263 /* Set Toeplitz hash value in the skb using the value from the
 264  * CQE (calculated by HW).
 265  */
 266 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 267                             const struct eth_fast_path_rx_cqe *cqe)
 268 {
 269         /* Set Toeplitz hash from CQE */
 270         if ((bp->dev->features & NETIF_F_RXHASH) &&
 271             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
 272                 return le32_to_cpu(cqe->rss_hash_result);
 273         return 0;
 274 }
 275
 276 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 277                             u16 cons, u16 prod,
 278                             struct eth_fast_path_rx_cqe *cqe)
 279 {
 280         struct bnx2x *bp = fp->bp;
 281         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 282         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 283         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 284         dma_addr_t mapping;
 285         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 286         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 287
 288         /* print error if current state != stop */
 289         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 290                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 291
 292         /* Try to map an empty data buffer from the aggregation info  */
 293         mapping = dma_map_single(&bp->pdev->dev,
 294                                  first_buf->data + NET_SKB_PAD,
 295                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 296         /*
 297          *  ...if it fails - move the skb from the consumer to the producer
 298          *  and set the current aggregation state as ERROR to drop it
 299          *  when TPA_STOP arrives.
 300          */
 301
 302         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 303                 /* Move the BD from the consumer to the producer */
 304                 bnx2x_reuse_rx_data(fp, cons, prod);
 305                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 306                 return;
 307         }
 308
 309         /* move empty data from pool to prod */
 310         prod_rx_buf->data = first_buf->data;
 311         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 312         /* point prod_bd to new data */
 313         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 314         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 315
 316         /* move partial skb from cons to pool (don't unmap yet) */
 317         *first_buf = *cons_rx_buf;
 318
 319         /* mark bin state as START */
 320         tpa_info->parsing_flags =
 321                 le16_to_cpu(cqe->pars_flags.flags);
 322         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 323         tpa_info->tpa_state = BNX2X_TPA_START;
 324         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 325         tpa_info->placement_offset = cqe->placement_offset;
 326         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe);
 327         if (fp->mode == TPA_MODE_GRO) {
 328                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 329                 tpa_info->full_page =
 330                         SGE_PAGE_SIZE * PAGES_PER_SGE / gro_size * gro_size;
 331                 /*
 332                  * FW 7.2.16 BUG workaround:
 333                  * if SGE size is (exactly) multiple gro_size
 334                  * fw will place one less frag on SGE.
 335                  * the calculation is done only for potentially
 336                  * dangerous MTUs.
 337                  */
 338                 if (unlikely(bp->gro_check))
 339                         if (!(SGE_PAGE_SIZE * PAGES_PER_SGE % gro_size))
 340                                 tpa_info->full_page -= gro_size;
 341                 tpa_info->gro_size = gro_size;
 342         }
 343
 344 #ifdef BNX2X_STOP_ON_ERROR
 345         fp->tpa_queue_used |= (1 << queue);
 346 #ifdef _ASM_GENERIC_INT_L64_H
 347         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 348 #else
 349         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 350 #endif
 351            fp->tpa_queue_used);
 352 #endif
 353 }
 354
 355 /* Timestamp option length allowed for TPA aggregation:
 356  *
 357  *              nop nop kind length echo val
 358  */
 359 #define TPA_TSTAMP_OPT_LEN      12
 360 /**
 361  * bnx2x_set_lro_mss - calculate the approximate value of the MSS
 362  *
 363  * @bp:                 driver handle
 364  * @parsing_flags:      parsing flags from the START CQE
 365  * @len_on_bd:          total length of the first packet for the
 366  *                      aggregation.
 367  *
 368  * Approximate value of the MSS for this aggregation calculated using
 369  * the first packet of it.
 370  */
 371 static inline u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
 372                                     u16 len_on_bd)
 373 {
 374         /*
 375          * TPA arrgregation won't have either IP options or TCP options
 376          * other than timestamp or IPv6 extension headers.
 377          */
 378         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 379
 380         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 381             PRS_FLAG_OVERETH_IPV6)
 382                 hdrs_len += sizeof(struct ipv6hdr);
 383         else /* IPv4 */
 384                 hdrs_len += sizeof(struct iphdr);
 385
 386
 387         /* Check if there was a TCP timestamp, if there is it's will
 388          * always be 12 bytes length: nop nop kind length echo val.
 389          *
 390          * Otherwise FW would close the aggregation.
 391          */
 392         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 393                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 394
 395         return len_on_bd - hdrs_len;
 396 }
 397
 398 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 399                                struct bnx2x_agg_info *tpa_info,
 400                                u16 pages,
 401                                struct sk_buff *skb,
 402                                struct eth_end_agg_rx_cqe *cqe,
 403                                u16 cqe_idx)
 404 {
 405         struct sw_rx_page *rx_pg, old_rx_pg;
 406         u32 i, frag_len, frag_size;
 407         int err, j, frag_id = 0;
 408         u16 len_on_bd = tpa_info->len_on_bd;
 409         u16 full_page = 0, gro_size = 0;
 410
 411         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 412
 413         if (fp->mode == TPA_MODE_GRO) {
 414                 gro_size = tpa_info->gro_size;
 415                 full_page = tpa_info->full_page;
 416         }
 417
 418         /* This is needed in order to enable forwarding support */
 419         if (frag_size) {
 420                 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
 421                                         tpa_info->parsing_flags, len_on_bd);
 422
 423                 /* set for GRO */
 424                 if (fp->mode == TPA_MODE_GRO)
 425                         skb_shinfo(skb)->gso_type =
 426                             (GET_FLAG(tpa_info->parsing_flags,
 427                                       PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 428                                                 PRS_FLAG_OVERETH_IPV6) ?
 429                                 SKB_GSO_TCPV6 : SKB_GSO_TCPV4;
 430         }
 431
 432
 433 #ifdef BNX2X_STOP_ON_ERROR
 434         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 435                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 436                           pages, cqe_idx);
 437                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 438                 bnx2x_panic();
 439                 return -EINVAL;
 440         }
 441 #endif
 442
 443         /* Run through the SGL and compose the fragmented skb */
 444         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 445                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 446
 447                 /* FW gives the indices of the SGE as if the ring is an array
 448                    (meaning that "next" element will consume 2 indices) */
 449                 if (fp->mode == TPA_MODE_GRO)
 450                         frag_len = min_t(u32, frag_size, (u32)full_page);
 451                 else /* LRO */
 452                         frag_len = min_t(u32, frag_size,
 453                                          (u32)(SGE_PAGE_SIZE * PAGES_PER_SGE));
 454
 455                 rx_pg = &fp->rx_page_ring[sge_idx];
 456                 old_rx_pg = *rx_pg;
 457
 458                 /* If we fail to allocate a substitute page, we simply stop
 459                    where we are and drop the whole packet */
 460                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 461                 if (unlikely(err)) {
 462                         fp->eth_q_stats.rx_skb_alloc_failed++;
 463                         return err;
 464                 }
 465
 466                 /* Unmap the page as we r going to pass it to the stack */
 467                 dma_unmap_page(&bp->pdev->dev,
 468                                dma_unmap_addr(&old_rx_pg, mapping),
 469                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 470                 /* Add one frag and update the appropriate fields in the skb */
 471                 if (fp->mode == TPA_MODE_LRO)
 472                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 473                 else { /* GRO */
 474                         int rem;
 475                         int offset = 0;
 476                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 477                                 int len = rem > gro_size ? gro_size : rem;
 478                                 skb_fill_page_desc(skb, frag_id++,
 479                                                    old_rx_pg.page, offset, len);
 480                                 if (offset)
 481                                         get_page(old_rx_pg.page);
 482                                 offset += len;
 483                         }
 484                 }
 485
 486                 skb->data_len += frag_len;
 487                 skb->truesize += SGE_PAGE_SIZE * PAGES_PER_SGE;
 488                 skb->len += frag_len;
 489
 490                 frag_size -= frag_len;
 491         }
 492
 493         return 0;
 494 }
 495
 496 static inline void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 497                                   struct bnx2x_agg_info *tpa_info,
 498                                   u16 pages,
 499                                   struct eth_end_agg_rx_cqe *cqe,
 500                                   u16 cqe_idx)
 501 {
 502         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 503         u8 pad = tpa_info->placement_offset;
 504         u16 len = tpa_info->len_on_bd;
 505         struct sk_buff *skb = NULL;
 506         u8 *new_data, *data = rx_buf->data;
 507         u8 old_tpa_state = tpa_info->tpa_state;
 508
 509         tpa_info->tpa_state = BNX2X_TPA_STOP;
 510
 511         /* If we there was an error during the handling of the TPA_START -
 512          * drop this aggregation.
 513          */
 514         if (old_tpa_state == BNX2X_TPA_ERROR)
 515                 goto drop;
 516
 517         /* Try to allocate the new data */
 518         new_data = kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
 519
 520         /* Unmap skb in the pool anyway, as we are going to change
 521            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 522            fails. */
 523         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 524                          fp->rx_buf_size, DMA_FROM_DEVICE);
 525         if (likely(new_data))
 526                 skb = build_skb(data);
 527
 528         if (likely(skb)) {
 529 #ifdef BNX2X_STOP_ON_ERROR
 530                 if (pad + len > fp->rx_buf_size) {
 531                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 532                                   pad, len, fp->rx_buf_size);
 533                         bnx2x_panic();
 534                         return;
 535                 }
 536 #endif
 537
 538                 skb_reserve(skb, pad + NET_SKB_PAD);
 539                 skb_put(skb, len);
 540                 skb->rxhash = tpa_info->rxhash;
 541
 542                 skb->protocol = eth_type_trans(skb, bp->dev);
 543                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 544
 545                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 546                                          skb, cqe, cqe_idx)) {
 547                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 548                                 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
 549                         napi_gro_receive(&fp->napi, skb);
 550                 } else {
 551                         DP(NETIF_MSG_RX_STATUS,
 552                            "Failed to allocate new pages - dropping packet!\n");
 553                         dev_kfree_skb_any(skb);
 554                 }
 555
 556
 557                 /* put new data in bin */
 558                 rx_buf->data = new_data;
 559
 560                 return;
 561         }
 562         kfree(new_data);
 563 drop:
 564         /* drop the packet and keep the buffer in the bin */
 565         DP(NETIF_MSG_RX_STATUS,
 566            "Failed to allocate or map a new skb - dropping packet!\n");
 567         fp->eth_q_stats.rx_skb_alloc_failed++;
 568 }
 569
 570
 571 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 572 {
 573         struct bnx2x *bp = fp->bp;
 574         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 575         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 576         int rx_pkt = 0;
 577
 578 #ifdef BNX2X_STOP_ON_ERROR
 579         if (unlikely(bp->panic))
 580                 return 0;
 581 #endif
 582
 583         /* CQ "next element" is of the size of the regular element,
 584            that's why it's ok here */
 585         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 586         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 587                 hw_comp_cons++;
 588
 589         bd_cons = fp->rx_bd_cons;
 590         bd_prod = fp->rx_bd_prod;
 591         bd_prod_fw = bd_prod;
 592         sw_comp_cons = fp->rx_comp_cons;
 593         sw_comp_prod = fp->rx_comp_prod;
 594
 595         /* Memory barrier necessary as speculative reads of the rx
 596          * buffer can be ahead of the index in the status block
 597          */
 598         rmb();
 599
 600         DP(NETIF_MSG_RX_STATUS,
 601            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 602            fp->index, hw_comp_cons, sw_comp_cons);
 603
 604         while (sw_comp_cons != hw_comp_cons) {
 605                 struct sw_rx_bd *rx_buf = NULL;
 606                 struct sk_buff *skb;
 607                 union eth_rx_cqe *cqe;
 608                 struct eth_fast_path_rx_cqe *cqe_fp;
 609                 u8 cqe_fp_flags;
 610                 enum eth_rx_cqe_type cqe_fp_type;
 611                 u16 len, pad, queue;
 612                 u8 *data;
 613
 614 #ifdef BNX2X_STOP_ON_ERROR
 615                 if (unlikely(bp->panic))
 616                         return 0;
 617 #endif
 618
 619                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 620                 bd_prod = RX_BD(bd_prod);
 621                 bd_cons = RX_BD(bd_cons);
 622
 623                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 624                 cqe_fp = &cqe->fast_path_cqe;
 625                 cqe_fp_flags = cqe_fp->type_error_flags;
 626                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 627
 628                 DP(NETIF_MSG_RX_STATUS,
 629                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 630                    CQE_TYPE(cqe_fp_flags),
 631                    cqe_fp_flags, cqe_fp->status_flags,
 632                    le32_to_cpu(cqe_fp->rss_hash_result),
 633                    le16_to_cpu(cqe_fp->vlan_tag),
 634                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 635
 636                 /* is this a slowpath msg? */
 637                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 638                         bnx2x_sp_event(fp, cqe);
 639                         goto next_cqe;
 640                 }
 641
 642                 rx_buf = &fp->rx_buf_ring[bd_cons];
 643                 data = rx_buf->data;
 644
 645                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 646                         struct bnx2x_agg_info *tpa_info;
 647                         u16 frag_size, pages;
 648 #ifdef BNX2X_STOP_ON_ERROR
 649                         /* sanity check */
 650                         if (fp->disable_tpa &&
 651                             (CQE_TYPE_START(cqe_fp_type) ||
 652                              CQE_TYPE_STOP(cqe_fp_type)))
 653                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 654                                           CQE_TYPE(cqe_fp_type));
 655 #endif
 656
 657                         if (CQE_TYPE_START(cqe_fp_type)) {
 658                                 u16 queue = cqe_fp->queue_index;
 659                                 DP(NETIF_MSG_RX_STATUS,
 660                                    "calling tpa_start on queue %d\n",
 661                                    queue);
 662
 663                                 bnx2x_tpa_start(fp, queue,
 664                                                 bd_cons, bd_prod,
 665                                                 cqe_fp);
 666
 667                                 goto next_rx;
 668
 669                         }
 670                         queue = cqe->end_agg_cqe.queue_index;
 671                         tpa_info = &fp->tpa_info[queue];
 672                         DP(NETIF_MSG_RX_STATUS,
 673                            "calling tpa_stop on queue %d\n",
 674                            queue);
 675
 676                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 677                                     tpa_info->len_on_bd;
 678
 679                         if (fp->mode == TPA_MODE_GRO)
 680                                 pages = (frag_size + tpa_info->full_page - 1) /
 681                                          tpa_info->full_page;
 682                         else
 683                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 684                                         SGE_PAGE_SHIFT;
 685
 686                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 687                                        &cqe->end_agg_cqe, comp_ring_cons);
 688 #ifdef BNX2X_STOP_ON_ERROR
 689                         if (bp->panic)
 690                                 return 0;
 691 #endif
 692
 693                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 694                         goto next_cqe;
 695                 }
 696                 /* non TPA */
 697                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 698                 pad = cqe_fp->placement_offset;
 699                 dma_sync_single_for_cpu(&bp->pdev->dev,
 700                                         dma_unmap_addr(rx_buf, mapping),
 701                                         pad + RX_COPY_THRESH,
 702                                         DMA_FROM_DEVICE);
 703                 pad += NET_SKB_PAD;
 704                 prefetch(data + pad); /* speedup eth_type_trans() */
 705                 /* is this an error packet? */
 706                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 707                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 708                            "ERROR  flags %x  rx packet %u\n",
 709                            cqe_fp_flags, sw_comp_cons);
 710                         fp->eth_q_stats.rx_err_discard_pkt++;
 711                         goto reuse_rx;
 712                 }
 713
 714                 /* Since we don't have a jumbo ring
 715                  * copy small packets if mtu > 1500
 716                  */
 717                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 718                     (len <= RX_COPY_THRESH)) {
 719                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 720                         if (skb == NULL) {
 721                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 722                                    "ERROR  packet dropped because of alloc failure\n");
 723                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 724                                 goto reuse_rx;
 725                         }
 726                         memcpy(skb->data, data + pad, len);
 727                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 728                 } else {
 729                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
 730                                 dma_unmap_single(&bp->pdev->dev,
 731                                                  dma_unmap_addr(rx_buf, mapping),
 732                                                  fp->rx_buf_size,
 733                                                  DMA_FROM_DEVICE);
 734                                 skb = build_skb(data);
 735                                 if (unlikely(!skb)) {
 736                                         kfree(data);
 737                                         fp->eth_q_stats.rx_skb_alloc_failed++;
 738                                         goto next_rx;
 739                                 }
 740                                 skb_reserve(skb, pad);
 741                         } else {
 742                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 743                                    "ERROR  packet dropped because of alloc failure\n");
 744                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 745 reuse_rx:
 746                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 747                                 goto next_rx;
 748                         }
 749                 }
 750
 751                 skb_put(skb, len);
 752                 skb->protocol = eth_type_trans(skb, bp->dev);
 753
 754                 /* Set Toeplitz hash for a none-LRO skb */
 755                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp);
 756
 757                 skb_checksum_none_assert(skb);
 758
 759                 if (bp->dev->features & NETIF_F_RXCSUM) {
 760
 761                         if (likely(BNX2X_RX_CSUM_OK(cqe)))
 762                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 763                         else
 764                                 fp->eth_q_stats.hw_csum_err++;
 765                 }
 766
 767                 skb_record_rx_queue(skb, fp->rx_queue);
 768
 769                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
 770                     PARSING_FLAGS_VLAN)
 771                         __vlan_hwaccel_put_tag(skb,
 772                                                le16_to_cpu(cqe_fp->vlan_tag));
 773                 napi_gro_receive(&fp->napi, skb);
 774
 775
 776 next_rx:
 777                 rx_buf->data = NULL;
 778
 779                 bd_cons = NEXT_RX_IDX(bd_cons);
 780                 bd_prod = NEXT_RX_IDX(bd_prod);
 781                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 782                 rx_pkt++;
 783 next_cqe:
 784                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 785                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 786
 787                 if (rx_pkt == budget)
 788                         break;
 789         } /* while */
 790
 791         fp->rx_bd_cons = bd_cons;
 792         fp->rx_bd_prod = bd_prod_fw;
 793         fp->rx_comp_cons = sw_comp_cons;
 794         fp->rx_comp_prod = sw_comp_prod;
 795
 796         /* Update producers */
 797         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 798                              fp->rx_sge_prod);
 799
 800         fp->rx_pkt += rx_pkt;
 801         fp->rx_calls++;
 802
 803         return rx_pkt;
 804 }
 805
 806 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 807 {
 808         struct bnx2x_fastpath *fp = fp_cookie;
 809         struct bnx2x *bp = fp->bp;
 810         u8 cos;
 811
 812         DP(NETIF_MSG_INTR,
 813            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
 814            fp->index, fp->fw_sb_id, fp->igu_sb_id);
 815         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 816
 817 #ifdef BNX2X_STOP_ON_ERROR
 818         if (unlikely(bp->panic))
 819                 return IRQ_HANDLED;
 820 #endif
 821
 822         /* Handle Rx and Tx according to MSI-X vector */
 823         prefetch(fp->rx_cons_sb);
 824
 825         for_each_cos_in_tx_queue(fp, cos)
 826                 prefetch(fp->txdata[cos].tx_cons_sb);
 827
 828         prefetch(&fp->sb_running_index[SM_RX_ID]);
 829         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 830
 831         return IRQ_HANDLED;
 832 }
 833
 834 /* HW Lock for shared dual port PHYs */
 835 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 836 {
 837         mutex_lock(&bp->port.phy_mutex);
 838
 839         if (bp->port.need_hw_lock)
 840                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 841 }
 842
 843 void bnx2x_release_phy_lock(struct bnx2x *bp)
 844 {
 845         if (bp->port.need_hw_lock)
 846                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 847
 848         mutex_unlock(&bp->port.phy_mutex);
 849 }
 850
 851 /* calculates MF speed according to current linespeed and MF configuration */
 852 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
 853 {
 854         u16 line_speed = bp->link_vars.line_speed;
 855         if (IS_MF(bp)) {
 856                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
 857                                                    bp->mf_config[BP_VN(bp)]);
 858
 859                 /* Calculate the current MAX line speed limit for the MF
 860                  * devices
 861                  */
 862                 if (IS_MF_SI(bp))
 863                         line_speed = (line_speed * maxCfg) / 100;
 864                 else { /* SD mode */
 865                         u16 vn_max_rate = maxCfg * 100;
 866
 867                         if (vn_max_rate < line_speed)
 868                                 line_speed = vn_max_rate;
 869                 }
 870         }
 871
 872         return line_speed;
 873 }
 874
 875 /**
 876  * bnx2x_fill_report_data - fill link report data to report
 877  *
 878  * @bp:         driver handle
 879  * @data:       link state to update
 880  *
 881  * It uses a none-atomic bit operations because is called under the mutex.
 882  */
 883 static inline void bnx2x_fill_report_data(struct bnx2x *bp,
 884                                           struct bnx2x_link_report_data *data)
 885 {
 886         u16 line_speed = bnx2x_get_mf_speed(bp);
 887
 888         memset(data, 0, sizeof(*data));
 889
 890         /* Fill the report data: efective line speed */
 891         data->line_speed = line_speed;
 892
 893         /* Link is down */
 894         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
 895                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 896                           &data->link_report_flags);
 897
 898         /* Full DUPLEX */
 899         if (bp->link_vars.duplex == DUPLEX_FULL)
 900                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
 901
 902         /* Rx Flow Control is ON */
 903         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
 904                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
 905
 906         /* Tx Flow Control is ON */
 907         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
 908                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
 909 }
 910
 911 /**
 912  * bnx2x_link_report - report link status to OS.
 913  *
 914  * @bp:         driver handle
 915  *
 916  * Calls the __bnx2x_link_report() under the same locking scheme
 917  * as a link/PHY state managing code to ensure a consistent link
 918  * reporting.
 919  */
 920
 921 void bnx2x_link_report(struct bnx2x *bp)
 922 {
 923         bnx2x_acquire_phy_lock(bp);
 924         __bnx2x_link_report(bp);
 925         bnx2x_release_phy_lock(bp);
 926 }
 927
 928 /**
 929  * __bnx2x_link_report - report link status to OS.
 930  *
 931  * @bp:         driver handle
 932  *
 933  * None atomic inmlementation.
 934  * Should be called under the phy_lock.
 935  */
 936 void __bnx2x_link_report(struct bnx2x *bp)
 937 {
 938         struct bnx2x_link_report_data cur_data;
 939
 940         /* reread mf_cfg */
 941         if (!CHIP_IS_E1(bp))
 942                 bnx2x_read_mf_cfg(bp);
 943
 944         /* Read the current link report info */
 945         bnx2x_fill_report_data(bp, &cur_data);
 946
 947         /* Don't report link down or exactly the same link status twice */
 948         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
 949             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 950                       &bp->last_reported_link.link_report_flags) &&
 951              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 952                       &cur_data.link_report_flags)))
 953                 return;
 954
 955         bp->link_cnt++;
 956
 957         /* We are going to report a new link parameters now -
 958          * remember the current data for the next time.
 959          */
 960         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
 961
 962         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 963                      &cur_data.link_report_flags)) {
 964                 netif_carrier_off(bp->dev);
 965                 netdev_err(bp->dev, "NIC Link is Down\n");
 966                 return;
 967         } else {
 968                 const char *duplex;
 969                 const char *flow;
 970
 971                 netif_carrier_on(bp->dev);
 972
 973                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
 974                                        &cur_data.link_report_flags))
 975                         duplex = "full";
 976                 else
 977                         duplex = "half";
 978
 979                 /* Handle the FC at the end so that only these flags would be
 980                  * possibly set. This way we may easily check if there is no FC
 981                  * enabled.
 982                  */
 983                 if (cur_data.link_report_flags) {
 984                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
 985                                      &cur_data.link_report_flags)) {
 986                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
 987                                      &cur_data.link_report_flags))
 988                                         flow = "ON - receive & transmit";
 989                                 else
 990                                         flow = "ON - receive";
 991                         } else {
 992                                 flow = "ON - transmit";
 993                         }
 994                 } else {
 995                         flow = "none";
 996                 }
 997                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
 998                             cur_data.line_speed, duplex, flow);
 999         }
1000 }
1001
1002 void bnx2x_init_rx_rings(struct bnx2x *bp)
1003 {
1004         int func = BP_FUNC(bp);
1005         u16 ring_prod;
1006         int i, j;
1007
1008         /* Allocate TPA resources */
1009         for_each_rx_queue(bp, j) {
1010                 struct bnx2x_fastpath *fp = &bp->fp[j];
1011
1012                 DP(NETIF_MSG_IFUP,
1013                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1014
1015                 if (!fp->disable_tpa) {
1016                         /* Fill the per-aggregtion pool */
1017                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1018                                 struct bnx2x_agg_info *tpa_info =
1019                                         &fp->tpa_info[i];
1020                                 struct sw_rx_bd *first_buf =
1021                                         &tpa_info->first_buf;
1022
1023                                 first_buf->data = kmalloc(fp->rx_buf_size + NET_SKB_PAD,
1024                                                           GFP_ATOMIC);
1025                                 if (!first_buf->data) {
1026                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1027                                                   j);
1028                                         bnx2x_free_tpa_pool(bp, fp, i);
1029                                         fp->disable_tpa = 1;
1030                                         break;
1031                                 }
1032                                 dma_unmap_addr_set(first_buf, mapping, 0);
1033                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1034                         }
1035
1036                         /* "next page" elements initialization */
1037                         bnx2x_set_next_page_sgl(fp);
1038
1039                         /* set SGEs bit mask */
1040                         bnx2x_init_sge_ring_bit_mask(fp);
1041
1042                         /* Allocate SGEs and initialize the ring elements */
1043                         for (i = 0, ring_prod = 0;
1044                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1045
1046                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1047                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1048                                                   i);
1049                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1050                                                   j);
1051                                         /* Cleanup already allocated elements */
1052                                         bnx2x_free_rx_sge_range(bp, fp,
1053                                                                 ring_prod);
1054                                         bnx2x_free_tpa_pool(bp, fp,
1055                                                             MAX_AGG_QS(bp));
1056                                         fp->disable_tpa = 1;
1057                                         ring_prod = 0;
1058                                         break;
1059                                 }
1060                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1061                         }
1062
1063                         fp->rx_sge_prod = ring_prod;
1064                 }
1065         }
1066
1067         for_each_rx_queue(bp, j) {
1068                 struct bnx2x_fastpath *fp = &bp->fp[j];
1069
1070                 fp->rx_bd_cons = 0;
1071
1072                 /* Activate BD ring */
1073                 /* Warning!
1074                  * this will generate an interrupt (to the TSTORM)
1075                  * must only be done after chip is initialized
1076                  */
1077                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1078                                      fp->rx_sge_prod);
1079
1080                 if (j != 0)
1081                         continue;
1082
1083                 if (CHIP_IS_E1(bp)) {
1084                         REG_WR(bp, BAR_USTRORM_INTMEM +
1085                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1086                                U64_LO(fp->rx_comp_mapping));
1087                         REG_WR(bp, BAR_USTRORM_INTMEM +
1088                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1089                                U64_HI(fp->rx_comp_mapping));
1090                 }
1091         }
1092 }
1093
1094 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1095 {
1096         int i;
1097         u8 cos;
1098
1099         for_each_tx_queue(bp, i) {
1100                 struct bnx2x_fastpath *fp = &bp->fp[i];
1101                 for_each_cos_in_tx_queue(fp, cos) {
1102                         struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
1103                         unsigned pkts_compl = 0, bytes_compl = 0;
1104
1105                         u16 sw_prod = txdata->tx_pkt_prod;
1106                         u16 sw_cons = txdata->tx_pkt_cons;
1107
1108                         while (sw_cons != sw_prod) {
1109                                 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1110                                     &pkts_compl, &bytes_compl);
1111                                 sw_cons++;
1112                         }
1113                         netdev_tx_reset_queue(
1114                             netdev_get_tx_queue(bp->dev, txdata->txq_index));
1115                 }
1116         }
1117 }
1118
1119 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1120 {
1121         struct bnx2x *bp = fp->bp;
1122         int i;
1123
1124         /* ring wasn't allocated */
1125         if (fp->rx_buf_ring == NULL)
1126                 return;
1127
1128         for (i = 0; i < NUM_RX_BD; i++) {
1129                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1130                 u8 *data = rx_buf->data;
1131
1132                 if (data == NULL)
1133                         continue;
1134                 dma_unmap_single(&bp->pdev->dev,
1135                                  dma_unmap_addr(rx_buf, mapping),
1136                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1137
1138                 rx_buf->data = NULL;
1139                 kfree(data);
1140         }
1141 }
1142
1143 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1144 {
1145         int j;
1146
1147         for_each_rx_queue(bp, j) {
1148                 struct bnx2x_fastpath *fp = &bp->fp[j];
1149
1150                 bnx2x_free_rx_bds(fp);
1151
1152                 if (!fp->disable_tpa)
1153                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1154         }
1155 }
1156
1157 void bnx2x_free_skbs(struct bnx2x *bp)
1158 {
1159         bnx2x_free_tx_skbs(bp);
1160         bnx2x_free_rx_skbs(bp);
1161 }
1162
1163 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1164 {
1165         /* load old values */
1166         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1167
1168         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1169                 /* leave all but MAX value */
1170                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1171
1172                 /* set new MAX value */
1173                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1174                                 & FUNC_MF_CFG_MAX_BW_MASK;
1175
1176                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1177         }
1178 }
1179
1180 /**
1181  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1182  *
1183  * @bp:         driver handle
1184  * @nvecs:      number of vectors to be released
1185  */
1186 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1187 {
1188         int i, offset = 0;
1189
1190         if (nvecs == offset)
1191                 return;
1192         free_irq(bp->msix_table[offset].vector, bp->dev);
1193         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1194            bp->msix_table[offset].vector);
1195         offset++;
1196 #ifdef BCM_CNIC
1197         if (nvecs == offset)
1198                 return;
1199         offset++;
1200 #endif
1201
1202         for_each_eth_queue(bp, i) {
1203                 if (nvecs == offset)
1204                         return;
1205                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1206                    i, bp->msix_table[offset].vector);
1207
1208                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1209         }
1210 }
1211
1212 void bnx2x_free_irq(struct bnx2x *bp)
1213 {
1214         if (bp->flags & USING_MSIX_FLAG &&
1215             !(bp->flags & USING_SINGLE_MSIX_FLAG))
1216                 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
1217                                      CNIC_PRESENT + 1);
1218         else
1219                 free_irq(bp->dev->irq, bp->dev);
1220 }
1221
1222 int __devinit bnx2x_enable_msix(struct bnx2x *bp)
1223 {
1224         int msix_vec = 0, i, rc, req_cnt;
1225
1226         bp->msix_table[msix_vec].entry = msix_vec;
1227         BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1228            bp->msix_table[0].entry);
1229         msix_vec++;
1230
1231 #ifdef BCM_CNIC
1232         bp->msix_table[msix_vec].entry = msix_vec;
1233         BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1234            bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1235         msix_vec++;
1236 #endif
1237         /* We need separate vectors for ETH queues only (not FCoE) */
1238         for_each_eth_queue(bp, i) {
1239                 bp->msix_table[msix_vec].entry = msix_vec;
1240                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1241                                msix_vec, msix_vec, i);
1242                 msix_vec++;
1243         }
1244
1245         req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_PRESENT + 1;
1246
1247         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1248
1249         /*
1250          * reconfigure number of tx/rx queues according to available
1251          * MSI-X vectors
1252          */
1253         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1254                 /* how less vectors we will have? */
1255                 int diff = req_cnt - rc;
1256
1257                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1258
1259                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1260
1261                 if (rc) {
1262                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1263                         goto no_msix;
1264                 }
1265                 /*
1266                  * decrease number of queues by number of unallocated entries
1267                  */
1268                 bp->num_queues -= diff;
1269
1270                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1271                                bp->num_queues);
1272         } else if (rc > 0) {
1273                 /* Get by with single vector */
1274                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1275                 if (rc) {
1276                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1277                                        rc);
1278                         goto no_msix;
1279                 }
1280
1281                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1282                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1283
1284         } else if (rc < 0) {
1285                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1286                 goto no_msix;
1287         }
1288
1289         bp->flags |= USING_MSIX_FLAG;
1290
1291         return 0;
1292
1293 no_msix:
1294         /* fall to INTx if not enough memory */
1295         if (rc == -ENOMEM)
1296                 bp->flags |= DISABLE_MSI_FLAG;
1297
1298         return rc;
1299 }
1300
1301 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1302 {
1303         int i, rc, offset = 0;
1304
1305         rc = request_irq(bp->msix_table[offset++].vector,
1306                          bnx2x_msix_sp_int, 0,
1307                          bp->dev->name, bp->dev);
1308         if (rc) {
1309                 BNX2X_ERR("request sp irq failed\n");
1310                 return -EBUSY;
1311         }
1312
1313 #ifdef BCM_CNIC
1314         offset++;
1315 #endif
1316         for_each_eth_queue(bp, i) {
1317                 struct bnx2x_fastpath *fp = &bp->fp[i];
1318                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1319                          bp->dev->name, i);
1320
1321                 rc = request_irq(bp->msix_table[offset].vector,
1322                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1323                 if (rc) {
1324                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1325                               bp->msix_table[offset].vector, rc);
1326                         bnx2x_free_msix_irqs(bp, offset);
1327                         return -EBUSY;
1328                 }
1329
1330                 offset++;
1331         }
1332
1333         i = BNX2X_NUM_ETH_QUEUES(bp);
1334         offset = 1 + CNIC_PRESENT;
1335         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1336                bp->msix_table[0].vector,
1337                0, bp->msix_table[offset].vector,
1338                i - 1, bp->msix_table[offset + i - 1].vector);
1339
1340         return 0;
1341 }
1342
1343 int bnx2x_enable_msi(struct bnx2x *bp)
1344 {
1345         int rc;
1346
1347         rc = pci_enable_msi(bp->pdev);
1348         if (rc) {
1349                 BNX2X_DEV_INFO("MSI is not attainable\n");
1350                 return -1;
1351         }
1352         bp->flags |= USING_MSI_FLAG;
1353
1354         return 0;
1355 }
1356
1357 static int bnx2x_req_irq(struct bnx2x *bp)
1358 {
1359         unsigned long flags;
1360         unsigned int irq;
1361
1362         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1363                 flags = 0;
1364         else
1365                 flags = IRQF_SHARED;
1366
1367         if (bp->flags & USING_MSIX_FLAG)
1368                 irq = bp->msix_table[0].vector;
1369         else
1370                 irq = bp->pdev->irq;
1371
1372         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1373 }
1374
1375 static inline int bnx2x_setup_irqs(struct bnx2x *bp)
1376 {
1377         int rc = 0;
1378         if (bp->flags & USING_MSIX_FLAG &&
1379             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1380                 rc = bnx2x_req_msix_irqs(bp);
1381                 if (rc)
1382                         return rc;
1383         } else {
1384                 bnx2x_ack_int(bp);
1385                 rc = bnx2x_req_irq(bp);
1386                 if (rc) {
1387                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1388                         return rc;
1389                 }
1390                 if (bp->flags & USING_MSI_FLAG) {
1391                         bp->dev->irq = bp->pdev->irq;
1392                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1393                                     bp->dev->irq);
1394                 }
1395                 if (bp->flags & USING_MSIX_FLAG) {
1396                         bp->dev->irq = bp->msix_table[0].vector;
1397                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1398                                     bp->dev->irq);
1399                 }
1400         }
1401
1402         return 0;
1403 }
1404
1405 static inline void bnx2x_napi_enable(struct bnx2x *bp)
1406 {
1407         int i;
1408
1409         for_each_rx_queue(bp, i)
1410                 napi_enable(&bnx2x_fp(bp, i, napi));
1411 }
1412
1413 static inline void bnx2x_napi_disable(struct bnx2x *bp)
1414 {
1415         int i;
1416
1417         for_each_rx_queue(bp, i)
1418                 napi_disable(&bnx2x_fp(bp, i, napi));
1419 }
1420
1421 void bnx2x_netif_start(struct bnx2x *bp)
1422 {
1423         if (netif_running(bp->dev)) {
1424                 bnx2x_napi_enable(bp);
1425                 bnx2x_int_enable(bp);
1426                 if (bp->state == BNX2X_STATE_OPEN)
1427                         netif_tx_wake_all_queues(bp->dev);
1428         }
1429 }
1430
1431 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1432 {
1433         bnx2x_int_disable_sync(bp, disable_hw);
1434         bnx2x_napi_disable(bp);
1435 }
1436
1437 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1438 {
1439         struct bnx2x *bp = netdev_priv(dev);
1440
1441 #ifdef BCM_CNIC
1442         if (!NO_FCOE(bp)) {
1443                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1444                 u16 ether_type = ntohs(hdr->h_proto);
1445
1446                 /* Skip VLAN tag if present */
1447                 if (ether_type == ETH_P_8021Q) {
1448                         struct vlan_ethhdr *vhdr =
1449                                 (struct vlan_ethhdr *)skb->data;
1450
1451                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1452                 }
1453
1454                 /* If ethertype is FCoE or FIP - use FCoE ring */
1455                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1456                         return bnx2x_fcoe_tx(bp, txq_index);
1457         }
1458 #endif
1459         /* select a non-FCoE queue */
1460         return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1461 }
1462
1463
1464 void bnx2x_set_num_queues(struct bnx2x *bp)
1465 {
1466         /* RSS queues */
1467         bp->num_queues = bnx2x_calc_num_queues(bp);
1468
1469 #ifdef BCM_CNIC
1470         /* override in STORAGE SD mode */
1471         if (IS_MF_STORAGE_SD(bp))
1472                 bp->num_queues = 1;
1473 #endif
1474         /* Add special queues */
1475         bp->num_queues += NON_ETH_CONTEXT_USE;
1476 }
1477
1478 /**
1479  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1480  *
1481  * @bp:         Driver handle
1482  *
1483  * We currently support for at most 16 Tx queues for each CoS thus we will
1484  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1485  * bp->max_cos.
1486  *
1487  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1488  * index after all ETH L2 indices.
1489  *
1490  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1491  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1492  * 16..31,...) with indicies that are not coupled with any real Tx queue.
1493  *
1494  * The proper configuration of skb->queue_mapping is handled by
1495  * bnx2x_select_queue() and __skb_tx_hash().
1496  *
1497  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1498  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1499  */
1500 static inline int bnx2x_set_real_num_queues(struct bnx2x *bp)
1501 {
1502         int rc, tx, rx;
1503
1504         tx = MAX_TXQS_PER_COS * bp->max_cos;
1505         rx = BNX2X_NUM_ETH_QUEUES(bp);
1506
1507 /* account for fcoe queue */
1508 #ifdef BCM_CNIC
1509         if (!NO_FCOE(bp)) {
1510                 rx += FCOE_PRESENT;
1511                 tx += FCOE_PRESENT;
1512         }
1513 #endif
1514
1515         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1516         if (rc) {
1517                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1518                 return rc;
1519         }
1520         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1521         if (rc) {
1522                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1523                 return rc;
1524         }
1525
1526         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1527                           tx, rx);
1528
1529         return rc;
1530 }
1531
1532 static inline void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1533 {
1534         int i;
1535
1536         for_each_queue(bp, i) {
1537                 struct bnx2x_fastpath *fp = &bp->fp[i];
1538                 u32 mtu;
1539
1540                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1541                 if (IS_FCOE_IDX(i))
1542                         /*
1543                          * Although there are no IP frames expected to arrive to
1544                          * this ring we still want to add an
1545                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1546                          * overrun attack.
1547                          */
1548                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1549                 else
1550                         mtu = bp->dev->mtu;
1551                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1552                                   IP_HEADER_ALIGNMENT_PADDING +
1553                                   ETH_OVREHEAD +
1554                                   mtu +
1555                                   BNX2X_FW_RX_ALIGN_END;
1556                 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1557         }
1558 }
1559
1560 static inline int bnx2x_init_rss_pf(struct bnx2x *bp)
1561 {
1562         int i;
1563         u8 ind_table[T_ETH_INDIRECTION_TABLE_SIZE] = {0};
1564         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1565
1566         /* Prepare the initial contents fo the indirection table if RSS is
1567          * enabled
1568          */
1569         for (i = 0; i < sizeof(ind_table); i++)
1570                 ind_table[i] =
1571                         bp->fp->cl_id +
1572                         ethtool_rxfh_indir_default(i, num_eth_queues);
1573
1574         /*
1575          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1576          * per-port, so if explicit configuration is needed , do it only
1577          * for a PMF.
1578          *
1579          * For 57712 and newer on the other hand it's a per-function
1580          * configuration.
1581          */
1582         return bnx2x_config_rss_eth(bp, ind_table,
1583                                     bp->port.pmf || !CHIP_IS_E1x(bp));
1584 }
1585
1586 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1587                         u8 *ind_table, bool config_hash)
1588 {
1589         struct bnx2x_config_rss_params params = {NULL};
1590         int i;
1591
1592         /* Although RSS is meaningless when there is a single HW queue we
1593          * still need it enabled in order to have HW Rx hash generated.
1594          *
1595          * if (!is_eth_multi(bp))
1596          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1597          */
1598
1599         params.rss_obj = rss_obj;
1600
1601         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1602
1603         __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1604
1605         /* RSS configuration */
1606         __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1607         __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1608         __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1609         __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1610
1611         /* Hash bits */
1612         params.rss_result_mask = MULTI_MASK;
1613
1614         memcpy(params.ind_table, ind_table, sizeof(params.ind_table));
1615
1616         if (config_hash) {
1617                 /* RSS keys */
1618                 for (i = 0; i < sizeof(params.rss_key) / 4; i++)
1619                         params.rss_key[i] = random32();
1620
1621                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1622         }
1623
1624         return bnx2x_config_rss(bp, &params);
1625 }
1626
1627 static inline int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1628 {
1629         struct bnx2x_func_state_params func_params = {NULL};
1630
1631         /* Prepare parameters for function state transitions */
1632         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1633
1634         func_params.f_obj = &bp->func_obj;
1635         func_params.cmd = BNX2X_F_CMD_HW_INIT;
1636
1637         func_params.params.hw_init.load_phase = load_code;
1638
1639         return bnx2x_func_state_change(bp, &func_params);
1640 }
1641
1642 /*
1643  * Cleans the object that have internal lists without sending
1644  * ramrods. Should be run when interrutps are disabled.
1645  */
1646 static void bnx2x_squeeze_objects(struct bnx2x *bp)
1647 {
1648         int rc;
1649         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1650         struct bnx2x_mcast_ramrod_params rparam = {NULL};
1651         struct bnx2x_vlan_mac_obj *mac_obj = &bp->fp->mac_obj;
1652
1653         /***************** Cleanup MACs' object first *************************/
1654
1655         /* Wait for completion of requested */
1656         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1657         /* Perform a dry cleanup */
1658         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1659
1660         /* Clean ETH primary MAC */
1661         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1662         rc = mac_obj->delete_all(bp, &bp->fp->mac_obj, &vlan_mac_flags,
1663                                  &ramrod_flags);
1664         if (rc != 0)
1665                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1666
1667         /* Cleanup UC list */
1668         vlan_mac_flags = 0;
1669         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1670         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1671                                  &ramrod_flags);
1672         if (rc != 0)
1673                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1674
1675         /***************** Now clean mcast object *****************************/
1676         rparam.mcast_obj = &bp->mcast_obj;
1677         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1678
1679         /* Add a DEL command... */
1680         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1681         if (rc < 0)
1682                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
1683                           rc);
1684
1685         /* ...and wait until all pending commands are cleared */
1686         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1687         while (rc != 0) {
1688                 if (rc < 0) {
1689                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1690                                   rc);
1691                         return;
1692                 }
1693
1694                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1695         }
1696 }
1697
1698 #ifndef BNX2X_STOP_ON_ERROR
1699 #define LOAD_ERROR_EXIT(bp, label) \
1700         do { \
1701                 (bp)->state = BNX2X_STATE_ERROR; \
1702                 goto label; \
1703         } while (0)
1704 #else
1705 #define LOAD_ERROR_EXIT(bp, label) \
1706         do { \
1707                 (bp)->state = BNX2X_STATE_ERROR; \
1708                 (bp)->panic = 1; \
1709                 return -EBUSY; \
1710         } while (0)
1711 #endif
1712
1713 bool bnx2x_test_firmware_version(struct bnx2x *bp, bool is_err)
1714 {
1715         /* build FW version dword */
1716         u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
1717                     (BCM_5710_FW_MINOR_VERSION << 8) +
1718                     (BCM_5710_FW_REVISION_VERSION << 16) +
1719                     (BCM_5710_FW_ENGINEERING_VERSION << 24);
1720
1721         /* read loaded FW from chip */
1722         u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
1723
1724         DP(NETIF_MSG_IFUP, "loaded fw %x, my fw %x\n", loaded_fw, my_fw);
1725
1726         if (loaded_fw != my_fw) {
1727                 if (is_err)
1728                         BNX2X_ERR("bnx2x with FW %x was already loaded, which mismatches my %x FW. aborting\n",
1729                                   loaded_fw, my_fw);
1730                 return false;
1731         }
1732
1733         return true;
1734 }
1735
1736 /* must be called with rtnl_lock */
1737 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1738 {
1739         int port = BP_PORT(bp);
1740         u32 load_code;
1741         int i, rc;
1742
1743 #ifdef BNX2X_STOP_ON_ERROR
1744         if (unlikely(bp->panic)) {
1745                 BNX2X_ERR("Can't load NIC when there is panic\n");
1746                 return -EPERM;
1747         }
1748 #endif
1749
1750         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1751
1752         /* Set the initial link reported state to link down */
1753         bnx2x_acquire_phy_lock(bp);
1754         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
1755         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1756                 &bp->last_reported_link.link_report_flags);
1757         bnx2x_release_phy_lock(bp);
1758
1759         /* must be called before memory allocation and HW init */
1760         bnx2x_ilt_set_info(bp);
1761
1762         /*
1763          * Zero fastpath structures preserving invariants like napi, which are
1764          * allocated only once, fp index, max_cos, bp pointer.
1765          * Also set fp->disable_tpa.
1766          */
1767         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
1768         for_each_queue(bp, i)
1769                 bnx2x_bz_fp(bp, i);
1770
1771
1772         /* Set the receive queues buffer size */
1773         bnx2x_set_rx_buf_size(bp);
1774
1775         if (bnx2x_alloc_mem(bp))
1776                 return -ENOMEM;
1777
1778         /* As long as bnx2x_alloc_mem() may possibly update
1779          * bp->num_queues, bnx2x_set_real_num_queues() should always
1780          * come after it.
1781          */
1782         rc = bnx2x_set_real_num_queues(bp);
1783         if (rc) {
1784                 BNX2X_ERR("Unable to set real_num_queues\n");
1785                 LOAD_ERROR_EXIT(bp, load_error0);
1786         }
1787
1788         /* configure multi cos mappings in kernel.
1789          * this configuration may be overriden by a multi class queue discipline
1790          * or by a dcbx negotiation result.
1791          */
1792         bnx2x_setup_tc(bp->dev, bp->max_cos);
1793
1794         bnx2x_napi_enable(bp);
1795
1796         /* set pf load just before approaching the MCP */
1797         bnx2x_set_pf_load(bp);
1798
1799         /* Send LOAD_REQUEST command to MCP
1800          * Returns the type of LOAD command:
1801          * if it is the first port to be initialized
1802          * common blocks should be initialized, otherwise - not
1803          */
1804         if (!BP_NOMCP(bp)) {
1805                 /* init fw_seq */
1806                 bp->fw_seq =
1807                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
1808                          DRV_MSG_SEQ_NUMBER_MASK);
1809                 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
1810
1811                 /* Get current FW pulse sequence */
1812                 bp->fw_drv_pulse_wr_seq =
1813                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
1814                          DRV_PULSE_SEQ_MASK);
1815                 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
1816
1817                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1818                 if (!load_code) {
1819                         BNX2X_ERR("MCP response failure, aborting\n");
1820                         rc = -EBUSY;
1821                         LOAD_ERROR_EXIT(bp, load_error1);
1822                 }
1823                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1824                         BNX2X_ERR("Driver load refused\n");
1825                         rc = -EBUSY; /* other port in diagnostic mode */
1826                         LOAD_ERROR_EXIT(bp, load_error1);
1827                 }
1828                 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
1829                     load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
1830                         /* abort nic load if version mismatch */
1831                         if (!bnx2x_test_firmware_version(bp, true)) {
1832                                 rc = -EBUSY;
1833                                 LOAD_ERROR_EXIT(bp, load_error2);
1834                         }
1835                 }
1836
1837         } else {
1838                 int path = BP_PATH(bp);
1839
1840                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
1841                    path, load_count[path][0], load_count[path][1],
1842                    load_count[path][2]);
1843                 load_count[path][0]++;
1844                 load_count[path][1 + port]++;
1845                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
1846                    path, load_count[path][0], load_count[path][1],
1847                    load_count[path][2]);
1848                 if (load_count[path][0] == 1)
1849                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1850                 else if (load_count[path][1 + port] == 1)
1851                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1852                 else
1853                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1854         }
1855
1856         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1857             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
1858             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
1859                 bp->port.pmf = 1;
1860                 /*
1861                  * We need the barrier to ensure the ordering between the
1862                  * writing to bp->port.pmf here and reading it from the
1863                  * bnx2x_periodic_task().
1864                  */
1865                 smp_mb();
1866         } else
1867                 bp->port.pmf = 0;
1868
1869         DP(NETIF_MSG_IFUP, "pmf %d\n", bp->port.pmf);
1870
1871         /* Init Function state controlling object */
1872         bnx2x__init_func_obj(bp);
1873
1874         /* Initialize HW */
1875         rc = bnx2x_init_hw(bp, load_code);
1876         if (rc) {
1877                 BNX2X_ERR("HW init failed, aborting\n");
1878                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1879                 LOAD_ERROR_EXIT(bp, load_error2);
1880         }
1881
1882         /* Connect to IRQs */
1883         rc = bnx2x_setup_irqs(bp);
1884         if (rc) {
1885                 BNX2X_ERR("IRQs setup failed\n");
1886                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1887                 LOAD_ERROR_EXIT(bp, load_error2);
1888         }
1889
1890         /* Setup NIC internals and enable interrupts */
1891         bnx2x_nic_init(bp, load_code);
1892
1893         /* Init per-function objects */
1894         bnx2x_init_bp_objs(bp);
1895
1896         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1897             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
1898             (bp->common.shmem2_base)) {
1899                 if (SHMEM2_HAS(bp, dcc_support))
1900                         SHMEM2_WR(bp, dcc_support,
1901                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1902                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1903         }
1904
1905         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1906         rc = bnx2x_func_start(bp);
1907         if (rc) {
1908                 BNX2X_ERR("Function start failed!\n");
1909                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1910                 LOAD_ERROR_EXIT(bp, load_error3);
1911         }
1912
1913         /* Send LOAD_DONE command to MCP */
1914         if (!BP_NOMCP(bp)) {
1915                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1916                 if (!load_code) {
1917                         BNX2X_ERR("MCP response failure, aborting\n");
1918                         rc = -EBUSY;
1919                         LOAD_ERROR_EXIT(bp, load_error3);
1920                 }
1921         }
1922
1923         rc = bnx2x_setup_leading(bp);
1924         if (rc) {
1925                 BNX2X_ERR("Setup leading failed!\n");
1926                 LOAD_ERROR_EXIT(bp, load_error3);
1927         }
1928
1929 #ifdef BCM_CNIC
1930         /* Enable Timer scan */
1931         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
1932 #endif
1933
1934         for_each_nondefault_queue(bp, i) {
1935                 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
1936                 if (rc) {
1937                         BNX2X_ERR("Queue setup failed\n");
1938                         LOAD_ERROR_EXIT(bp, load_error4);
1939                 }
1940         }
1941
1942         rc = bnx2x_init_rss_pf(bp);
1943         if (rc) {
1944                 BNX2X_ERR("PF RSS init failed\n");
1945                 LOAD_ERROR_EXIT(bp, load_error4);
1946         }
1947
1948         /* Now when Clients are configured we are ready to work */
1949         bp->state = BNX2X_STATE_OPEN;
1950
1951         /* Configure a ucast MAC */
1952         rc = bnx2x_set_eth_mac(bp, true);
1953         if (rc) {
1954                 BNX2X_ERR("Setting Ethernet MAC failed\n");
1955                 LOAD_ERROR_EXIT(bp, load_error4);
1956         }
1957
1958         if (bp->pending_max) {
1959                 bnx2x_update_max_mf_config(bp, bp->pending_max);
1960                 bp->pending_max = 0;
1961         }
1962
1963         if (bp->port.pmf)
1964                 bnx2x_initial_phy_init(bp, load_mode);
1965
1966         /* Start fast path */
1967
1968         /* Initialize Rx filter. */
1969         netif_addr_lock_bh(bp->dev);
1970         bnx2x_set_rx_mode(bp->dev);
1971         netif_addr_unlock_bh(bp->dev);
1972
1973         /* Start the Tx */
1974         switch (load_mode) {
1975         case LOAD_NORMAL:
1976                 /* Tx queue should be only reenabled */
1977                 netif_tx_wake_all_queues(bp->dev);
1978                 break;
1979
1980         case LOAD_OPEN:
1981                 netif_tx_start_all_queues(bp->dev);
1982                 smp_mb__after_clear_bit();
1983                 break;
1984
1985         case LOAD_DIAG:
1986                 bp->state = BNX2X_STATE_DIAG;
1987                 break;
1988
1989         default:
1990                 break;
1991         }
1992
1993         if (bp->port.pmf)
1994                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_DCB_CONFIGURED, 0);
1995         else
1996                 bnx2x__link_status_update(bp);
1997
1998         /* start the timer */
1999         mod_timer(&bp->timer, jiffies + bp->current_interval);
2000
2001 #ifdef BCM_CNIC
2002         /* re-read iscsi info */
2003         bnx2x_get_iscsi_info(bp);
2004         bnx2x_setup_cnic_irq_info(bp);
2005         if (bp->state == BNX2X_STATE_OPEN)
2006                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2007 #endif
2008
2009         /* mark driver is loaded in shmem2 */
2010         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2011                 u32 val;
2012                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2013                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2014                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2015                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2016         }
2017
2018         /* Wait for all pending SP commands to complete */
2019         if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2020                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2021                 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2022                 return -EBUSY;
2023         }
2024
2025         bnx2x_dcbx_init(bp);
2026         return 0;
2027
2028 #ifndef BNX2X_STOP_ON_ERROR
2029 load_error4:
2030 #ifdef BCM_CNIC
2031         /* Disable Timer scan */
2032         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2033 #endif
2034 load_error3:
2035         bnx2x_int_disable_sync(bp, 1);
2036
2037         /* Clean queueable objects */
2038         bnx2x_squeeze_objects(bp);
2039
2040         /* Free SKBs, SGEs, TPA pool and driver internals */
2041         bnx2x_free_skbs(bp);
2042         for_each_rx_queue(bp, i)
2043                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2044
2045         /* Release IRQs */
2046         bnx2x_free_irq(bp);
2047 load_error2:
2048         if (!BP_NOMCP(bp)) {
2049                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2050                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2051         }
2052
2053         bp->port.pmf = 0;
2054 load_error1:
2055         bnx2x_napi_disable(bp);
2056         /* clear pf_load status, as it was already set */
2057         bnx2x_clear_pf_load(bp);
2058 load_error0:
2059         bnx2x_free_mem(bp);
2060
2061         return rc;
2062 #endif /* ! BNX2X_STOP_ON_ERROR */
2063 }
2064
2065 /* must be called with rtnl_lock */
2066 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
2067 {
2068         int i;
2069         bool global = false;
2070
2071         /* mark driver is unloaded in shmem2 */
2072         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2073                 u32 val;
2074                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2075                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2076                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2077         }
2078
2079         if ((bp->state == BNX2X_STATE_CLOSED) ||
2080             (bp->state == BNX2X_STATE_ERROR)) {
2081                 /* We can get here if the driver has been unloaded
2082                  * during parity error recovery and is either waiting for a
2083                  * leader to complete or for other functions to unload and
2084                  * then ifdown has been issued. In this case we want to
2085                  * unload and let other functions to complete a recovery
2086                  * process.
2087                  */
2088                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2089                 bp->is_leader = 0;
2090                 bnx2x_release_leader_lock(bp);
2091                 smp_mb();
2092
2093                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2094                 BNX2X_ERR("Can't unload in closed or error state\n");
2095                 return -EINVAL;
2096         }
2097
2098         /*
2099          * It's important to set the bp->state to the value different from
2100          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2101          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2102          */
2103         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2104         smp_mb();
2105
2106         /* Stop Tx */
2107         bnx2x_tx_disable(bp);
2108
2109 #ifdef BCM_CNIC
2110         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2111 #endif
2112
2113         bp->rx_mode = BNX2X_RX_MODE_NONE;
2114
2115         del_timer_sync(&bp->timer);
2116
2117         /* Set ALWAYS_ALIVE bit in shmem */
2118         bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2119
2120         bnx2x_drv_pulse(bp);
2121
2122         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2123         bnx2x_save_statistics(bp);
2124
2125         /* Cleanup the chip if needed */
2126         if (unload_mode != UNLOAD_RECOVERY)
2127                 bnx2x_chip_cleanup(bp, unload_mode);
2128         else {
2129                 /* Send the UNLOAD_REQUEST to the MCP */
2130                 bnx2x_send_unload_req(bp, unload_mode);
2131
2132                 /*
2133                  * Prevent transactions to host from the functions on the
2134                  * engine that doesn't reset global blocks in case of global
2135                  * attention once gloabl blocks are reset and gates are opened
2136                  * (the engine which leader will perform the recovery
2137                  * last).
2138                  */
2139                 if (!CHIP_IS_E1x(bp))
2140                         bnx2x_pf_disable(bp);
2141
2142                 /* Disable HW interrupts, NAPI */
2143                 bnx2x_netif_stop(bp, 1);
2144
2145                 /* Release IRQs */
2146                 bnx2x_free_irq(bp);
2147
2148                 /* Report UNLOAD_DONE to MCP */
2149                 bnx2x_send_unload_done(bp);
2150         }
2151
2152         /*
2153          * At this stage no more interrupts will arrive so we may safly clean
2154          * the queueable objects here in case they failed to get cleaned so far.
2155          */
2156         bnx2x_squeeze_objects(bp);
2157
2158         /* There should be no more pending SP commands at this stage */
2159         bp->sp_state = 0;
2160
2161         bp->port.pmf = 0;
2162
2163         /* Free SKBs, SGEs, TPA pool and driver internals */
2164         bnx2x_free_skbs(bp);
2165         for_each_rx_queue(bp, i)
2166                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2167
2168         bnx2x_free_mem(bp);
2169
2170         bp->state = BNX2X_STATE_CLOSED;
2171
2172         /* Check if there are pending parity attentions. If there are - set
2173          * RECOVERY_IN_PROGRESS.
2174          */
2175         if (bnx2x_chk_parity_attn(bp, &global, false)) {
2176                 bnx2x_set_reset_in_progress(bp);
2177
2178                 /* Set RESET_IS_GLOBAL if needed */
2179                 if (global)
2180                         bnx2x_set_reset_global(bp);
2181         }
2182
2183
2184         /* The last driver must disable a "close the gate" if there is no
2185          * parity attention or "process kill" pending.
2186          */
2187         if (!bnx2x_clear_pf_load(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
2188                 bnx2x_disable_close_the_gate(bp);
2189
2190         return 0;
2191 }
2192
2193 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2194 {
2195         u16 pmcsr;
2196
2197         /* If there is no power capability, silently succeed */
2198         if (!bp->pm_cap) {
2199                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2200                 return 0;
2201         }
2202
2203         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2204
2205         switch (state) {
2206         case PCI_D0:
2207                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2208                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2209                                        PCI_PM_CTRL_PME_STATUS));
2210
2211                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2212                         /* delay required during transition out of D3hot */
2213                         msleep(20);
2214                 break;
2215
2216         case PCI_D3hot:
2217                 /* If there are other clients above don't
2218                    shut down the power */
2219                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2220                         return 0;
2221                 /* Don't shut down the power for emulation and FPGA */
2222                 if (CHIP_REV_IS_SLOW(bp))
2223                         return 0;
2224
2225                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2226                 pmcsr |= 3;
2227
2228                 if (bp->wol)
2229                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2230
2231                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2232                                       pmcsr);
2233
2234                 /* No more memory access after this point until
2235                 * device is brought back to D0.
2236                 */
2237                 break;
2238
2239         default:
2240                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
2241                 return -EINVAL;
2242         }
2243         return 0;
2244 }
2245
2246 /*
2247  * net_device service functions
2248  */
2249 int bnx2x_poll(struct napi_struct *napi, int budget)
2250 {
2251         int work_done = 0;
2252         u8 cos;
2253         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2254                                                  napi);
2255         struct bnx2x *bp = fp->bp;
2256
2257         while (1) {
2258 #ifdef BNX2X_STOP_ON_ERROR
2259                 if (unlikely(bp->panic)) {
2260                         napi_complete(napi);
2261                         return 0;
2262                 }
2263 #endif
2264
2265                 for_each_cos_in_tx_queue(fp, cos)
2266                         if (bnx2x_tx_queue_has_work(&fp->txdata[cos]))
2267                                 bnx2x_tx_int(bp, &fp->txdata[cos]);
2268
2269
2270                 if (bnx2x_has_rx_work(fp)) {
2271                         work_done += bnx2x_rx_int(fp, budget - work_done);
2272
2273                         /* must not complete if we consumed full budget */
2274                         if (work_done >= budget)
2275                                 break;
2276                 }
2277
2278                 /* Fall out from the NAPI loop if needed */
2279                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2280 #ifdef BCM_CNIC
2281                         /* No need to update SB for FCoE L2 ring as long as
2282                          * it's connected to the default SB and the SB
2283                          * has been updated when NAPI was scheduled.
2284                          */
2285                         if (IS_FCOE_FP(fp)) {
2286                                 napi_complete(napi);
2287                                 break;
2288                         }
2289 #endif
2290
2291                         bnx2x_update_fpsb_idx(fp);
2292                         /* bnx2x_has_rx_work() reads the status block,
2293                          * thus we need to ensure that status block indices
2294                          * have been actually read (bnx2x_update_fpsb_idx)
2295                          * prior to this check (bnx2x_has_rx_work) so that
2296                          * we won't write the "newer" value of the status block
2297                          * to IGU (if there was a DMA right after
2298                          * bnx2x_has_rx_work and if there is no rmb, the memory
2299                          * reading (bnx2x_update_fpsb_idx) may be postponed
2300                          * to right before bnx2x_ack_sb). In this case there
2301                          * will never be another interrupt until there is
2302                          * another update of the status block, while there
2303                          * is still unhandled work.
2304                          */
2305                         rmb();
2306
2307                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2308                                 napi_complete(napi);
2309                                 /* Re-enable interrupts */
2310                                 DP(NETIF_MSG_RX_STATUS,
2311                                    "Update index to %d\n", fp->fp_hc_idx);
2312                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2313                                              le16_to_cpu(fp->fp_hc_idx),
2314                                              IGU_INT_ENABLE, 1);
2315                                 break;
2316                         }
2317                 }
2318         }
2319
2320         return work_done;
2321 }
2322
2323 /* we split the first BD into headers and data BDs
2324  * to ease the pain of our fellow microcode engineers
2325  * we use one mapping for both BDs
2326  * So far this has only been observed to happen
2327  * in Other Operating Systems(TM)
2328  */
2329 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
2330                                    struct bnx2x_fp_txdata *txdata,
2331                                    struct sw_tx_bd *tx_buf,
2332                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
2333                                    u16 bd_prod, int nbd)
2334 {
2335         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2336         struct eth_tx_bd *d_tx_bd;
2337         dma_addr_t mapping;
2338         int old_len = le16_to_cpu(h_tx_bd->nbytes);
2339
2340         /* first fix first BD */
2341         h_tx_bd->nbd = cpu_to_le16(nbd);
2342         h_tx_bd->nbytes = cpu_to_le16(hlen);
2343
2344         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x) nbd %d\n",
2345            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo, h_tx_bd->nbd);
2346
2347         /* now get a new data BD
2348          * (after the pbd) and fill it */
2349         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2350         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2351
2352         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2353                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2354
2355         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2356         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2357         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2358
2359         /* this marks the BD as one that has no individual mapping */
2360         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2361
2362         DP(NETIF_MSG_TX_QUEUED,
2363            "TSO split data size is %d (%x:%x)\n",
2364            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2365
2366         /* update tx_bd */
2367         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2368
2369         return bd_prod;
2370 }
2371
2372 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2373 {
2374         if (fix > 0)
2375                 csum = (u16) ~csum_fold(csum_sub(csum,
2376                                 csum_partial(t_header - fix, fix, 0)));
2377
2378         else if (fix < 0)
2379                 csum = (u16) ~csum_fold(csum_add(csum,
2380                                 csum_partial(t_header, -fix, 0)));
2381
2382         return swab16(csum);
2383 }
2384
2385 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2386 {
2387         u32 rc;
2388
2389         if (skb->ip_summed != CHECKSUM_PARTIAL)
2390                 rc = XMIT_PLAIN;
2391
2392         else {
2393                 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
2394                         rc = XMIT_CSUM_V6;
2395                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2396                                 rc |= XMIT_CSUM_TCP;
2397
2398                 } else {
2399                         rc = XMIT_CSUM_V4;
2400                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2401                                 rc |= XMIT_CSUM_TCP;
2402                 }
2403         }
2404
2405         if (skb_is_gso_v6(skb))
2406                 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2407         else if (skb_is_gso(skb))
2408                 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
2409
2410         return rc;
2411 }
2412
2413 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2414 /* check if packet requires linearization (packet is too fragmented)
2415    no need to check fragmentation if page size > 8K (there will be no
2416    violation to FW restrictions) */
2417 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2418                              u32 xmit_type)
2419 {
2420         int to_copy = 0;
2421         int hlen = 0;
2422         int first_bd_sz = 0;
2423
2424         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2425         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2426
2427                 if (xmit_type & XMIT_GSO) {
2428                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2429                         /* Check if LSO packet needs to be copied:
2430                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2431                         int wnd_size = MAX_FETCH_BD - 3;
2432                         /* Number of windows to check */
2433                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2434                         int wnd_idx = 0;
2435                         int frag_idx = 0;
2436                         u32 wnd_sum = 0;
2437
2438                         /* Headers length */
2439                         hlen = (int)(skb_transport_header(skb) - skb->data) +
2440                                 tcp_hdrlen(skb);
2441
2442                         /* Amount of data (w/o headers) on linear part of SKB*/
2443                         first_bd_sz = skb_headlen(skb) - hlen;
2444
2445                         wnd_sum  = first_bd_sz;
2446
2447                         /* Calculate the first sum - it's special */
2448                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2449                                 wnd_sum +=
2450                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
2451
2452                         /* If there was data on linear skb data - check it */
2453                         if (first_bd_sz > 0) {
2454                                 if (unlikely(wnd_sum < lso_mss)) {
2455                                         to_copy = 1;
2456                                         goto exit_lbl;
2457                                 }
2458
2459                                 wnd_sum -= first_bd_sz;
2460                         }
2461
2462                         /* Others are easier: run through the frag list and
2463                            check all windows */
2464                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2465                                 wnd_sum +=
2466                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
2467
2468                                 if (unlikely(wnd_sum < lso_mss)) {
2469                                         to_copy = 1;
2470                                         break;
2471                                 }
2472                                 wnd_sum -=
2473                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
2474                         }
2475                 } else {
2476                         /* in non-LSO too fragmented packet should always
2477                            be linearized */
2478                         to_copy = 1;
2479                 }
2480         }
2481
2482 exit_lbl:
2483         if (unlikely(to_copy))
2484                 DP(NETIF_MSG_TX_QUEUED,
2485                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
2486                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2487                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2488
2489         return to_copy;
2490 }
2491 #endif
2492
2493 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2494                                         u32 xmit_type)
2495 {
2496         *parsing_data |= (skb_shinfo(skb)->gso_size <<
2497                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2498                               ETH_TX_PARSE_BD_E2_LSO_MSS;
2499         if ((xmit_type & XMIT_GSO_V6) &&
2500             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2501                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
2502 }
2503
2504 /**
2505  * bnx2x_set_pbd_gso - update PBD in GSO case.
2506  *
2507  * @skb:        packet skb
2508  * @pbd:        parse BD
2509  * @xmit_type:  xmit flags
2510  */
2511 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2512                                      struct eth_tx_parse_bd_e1x *pbd,
2513                                      u32 xmit_type)
2514 {
2515         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2516         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2517         pbd->tcp_flags = pbd_tcp_flags(skb);
2518
2519         if (xmit_type & XMIT_GSO_V4) {
2520                 pbd->ip_id = swab16(ip_hdr(skb)->id);
2521                 pbd->tcp_pseudo_csum =
2522                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2523                                                   ip_hdr(skb)->daddr,
2524                                                   0, IPPROTO_TCP, 0));
2525
2526         } else
2527                 pbd->tcp_pseudo_csum =
2528                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2529                                                 &ipv6_hdr(skb)->daddr,
2530                                                 0, IPPROTO_TCP, 0));
2531
2532         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2533 }
2534
2535 /**
2536  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
2537  *
2538  * @bp:                 driver handle
2539  * @skb:                packet skb
2540  * @parsing_data:       data to be updated
2541  * @xmit_type:          xmit flags
2542  *
2543  * 57712 related
2544  */
2545 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
2546         u32 *parsing_data, u32 xmit_type)
2547 {
2548         *parsing_data |=
2549                         ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
2550                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
2551                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
2552
2553         if (xmit_type & XMIT_CSUM_TCP) {
2554                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
2555                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
2556                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
2557
2558                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
2559         } else
2560                 /* We support checksum offload for TCP and UDP only.
2561                  * No need to pass the UDP header length - it's a constant.
2562                  */
2563                 return skb_transport_header(skb) +
2564                                 sizeof(struct udphdr) - skb->data;
2565 }
2566
2567 static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2568         struct eth_tx_start_bd *tx_start_bd, u32 xmit_type)
2569 {
2570         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2571
2572         if (xmit_type & XMIT_CSUM_V4)
2573                 tx_start_bd->bd_flags.as_bitfield |=
2574                                         ETH_TX_BD_FLAGS_IP_CSUM;
2575         else
2576                 tx_start_bd->bd_flags.as_bitfield |=
2577                                         ETH_TX_BD_FLAGS_IPV6;
2578
2579         if (!(xmit_type & XMIT_CSUM_TCP))
2580                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
2581 }
2582
2583 /**
2584  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
2585  *
2586  * @bp:         driver handle
2587  * @skb:        packet skb
2588  * @pbd:        parse BD to be updated
2589  * @xmit_type:  xmit flags
2590  */
2591 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2592         struct eth_tx_parse_bd_e1x *pbd,
2593         u32 xmit_type)
2594 {
2595         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
2596
2597         /* for now NS flag is not used in Linux */
2598         pbd->global_data =
2599                 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
2600                          ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
2601
2602         pbd->ip_hlen_w = (skb_transport_header(skb) -
2603                         skb_network_header(skb)) >> 1;
2604
2605         hlen += pbd->ip_hlen_w;
2606
2607         /* We support checksum offload for TCP and UDP only */
2608         if (xmit_type & XMIT_CSUM_TCP)
2609                 hlen += tcp_hdrlen(skb) / 2;
2610         else
2611                 hlen += sizeof(struct udphdr) / 2;
2612
2613         pbd->total_hlen_w = cpu_to_le16(hlen);
2614         hlen = hlen*2;
2615
2616         if (xmit_type & XMIT_CSUM_TCP) {
2617                 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
2618
2619         } else {
2620                 s8 fix = SKB_CS_OFF(skb); /* signed! */
2621
2622                 DP(NETIF_MSG_TX_QUEUED,
2623                    "hlen %d  fix %d  csum before fix %x\n",
2624                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
2625
2626                 /* HW bug: fixup the CSUM */
2627                 pbd->tcp_pseudo_csum =
2628                         bnx2x_csum_fix(skb_transport_header(skb),
2629                                        SKB_CS(skb), fix);
2630
2631                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2632                    pbd->tcp_pseudo_csum);
2633         }
2634
2635         return hlen;
2636 }
2637
2638 /* called with netif_tx_lock
2639  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
2640  * netif_wake_queue()
2641  */
2642 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
2643 {
2644         struct bnx2x *bp = netdev_priv(dev);
2645
2646         struct bnx2x_fastpath *fp;
2647         struct netdev_queue *txq;
2648         struct bnx2x_fp_txdata *txdata;
2649         struct sw_tx_bd *tx_buf;
2650         struct eth_tx_start_bd *tx_start_bd, *first_bd;
2651         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
2652         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
2653         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2654         u32 pbd_e2_parsing_data = 0;
2655         u16 pkt_prod, bd_prod;
2656         int nbd, txq_index, fp_index, txdata_index;
2657         dma_addr_t mapping;
2658         u32 xmit_type = bnx2x_xmit_type(bp, skb);
2659         int i;
2660         u8 hlen = 0;
2661         __le16 pkt_size = 0;
2662         struct ethhdr *eth;
2663         u8 mac_type = UNICAST_ADDRESS;
2664
2665 #ifdef BNX2X_STOP_ON_ERROR
2666         if (unlikely(bp->panic))
2667                 return NETDEV_TX_BUSY;
2668 #endif
2669
2670         txq_index = skb_get_queue_mapping(skb);
2671         txq = netdev_get_tx_queue(dev, txq_index);
2672
2673         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + FCOE_PRESENT);
2674
2675         /* decode the fastpath index and the cos index from the txq */
2676         fp_index = TXQ_TO_FP(txq_index);
2677         txdata_index = TXQ_TO_COS(txq_index);
2678
2679 #ifdef BCM_CNIC
2680         /*
2681          * Override the above for the FCoE queue:
2682          *   - FCoE fp entry is right after the ETH entries.
2683          *   - FCoE L2 queue uses bp->txdata[0] only.
2684          */
2685         if (unlikely(!NO_FCOE(bp) && (txq_index ==
2686                                       bnx2x_fcoe_tx(bp, txq_index)))) {
2687                 fp_index = FCOE_IDX;
2688                 txdata_index = 0;
2689         }
2690 #endif
2691
2692         /* enable this debug print to view the transmission queue being used
2693         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
2694            txq_index, fp_index, txdata_index); */
2695
2696         /* locate the fastpath and the txdata */
2697         fp = &bp->fp[fp_index];
2698         txdata = &fp->txdata[txdata_index];
2699
2700         /* enable this debug print to view the tranmission details
2701         DP(NETIF_MSG_TX_QUEUED,
2702            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
2703            txdata->cid, fp_index, txdata_index, txdata, fp); */
2704
2705         if (unlikely(bnx2x_tx_avail(bp, txdata) <
2706                      (skb_shinfo(skb)->nr_frags + 3))) {
2707                 fp->eth_q_stats.driver_xoff++;
2708                 netif_tx_stop_queue(txq);
2709                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2710                 return NETDEV_TX_BUSY;
2711         }
2712
2713         DP(NETIF_MSG_TX_QUEUED,
2714            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x\n",
2715            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
2716            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
2717
2718         eth = (struct ethhdr *)skb->data;
2719
2720         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2721         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
2722                 if (is_broadcast_ether_addr(eth->h_dest))
2723                         mac_type = BROADCAST_ADDRESS;
2724                 else
2725                         mac_type = MULTICAST_ADDRESS;
2726         }
2727
2728 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2729         /* First, check if we need to linearize the skb (due to FW
2730            restrictions). No need to check fragmentation if page size > 8K
2731            (there will be no violation to FW restrictions) */
2732         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
2733                 /* Statistics of linearization */
2734                 bp->lin_cnt++;
2735                 if (skb_linearize(skb) != 0) {
2736                         DP(NETIF_MSG_TX_QUEUED,
2737                            "SKB linearization failed - silently dropping this SKB\n");
2738                         dev_kfree_skb_any(skb);
2739                         return NETDEV_TX_OK;
2740                 }
2741         }
2742 #endif
2743         /* Map skb linear data for DMA */
2744         mapping = dma_map_single(&bp->pdev->dev, skb->data,
2745                                  skb_headlen(skb), DMA_TO_DEVICE);
2746         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2747                 DP(NETIF_MSG_TX_QUEUED,
2748                    "SKB mapping failed - silently dropping this SKB\n");
2749                 dev_kfree_skb_any(skb);
2750                 return NETDEV_TX_OK;
2751         }
2752         /*
2753         Please read carefully. First we use one BD which we mark as start,
2754         then we have a parsing info BD (used for TSO or xsum),
2755         and only then we have the rest of the TSO BDs.
2756         (don't forget to mark the last one as last,
2757         and to unmap only AFTER you write to the BD ...)
2758         And above all, all pdb sizes are in words - NOT DWORDS!
2759         */
2760
2761         /* get current pkt produced now - advance it just before sending packet
2762          * since mapping of pages may fail and cause packet to be dropped
2763          */
2764         pkt_prod = txdata->tx_pkt_prod;
2765         bd_prod = TX_BD(txdata->tx_bd_prod);
2766
2767         /* get a tx_buf and first BD
2768          * tx_start_bd may be changed during SPLIT,
2769          * but first_bd will always stay first
2770          */
2771         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
2772         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
2773         first_bd = tx_start_bd;
2774
2775         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
2776         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
2777                  mac_type);
2778
2779         /* header nbd */
2780         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
2781
2782         /* remember the first BD of the packet */
2783         tx_buf->first_bd = txdata->tx_bd_prod;
2784         tx_buf->skb = skb;
2785         tx_buf->flags = 0;
2786
2787         DP(NETIF_MSG_TX_QUEUED,
2788            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
2789            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
2790
2791         if (vlan_tx_tag_present(skb)) {
2792                 tx_start_bd->vlan_or_ethertype =
2793                     cpu_to_le16(vlan_tx_tag_get(skb));
2794                 tx_start_bd->bd_flags.as_bitfield |=
2795                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
2796         } else
2797                 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
2798
2799         /* turn on parsing and get a BD */
2800         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2801
2802         if (xmit_type & XMIT_CSUM)
2803                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
2804
2805         if (!CHIP_IS_E1x(bp)) {
2806                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
2807                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
2808                 /* Set PBD in checksum offload case */
2809                 if (xmit_type & XMIT_CSUM)
2810                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
2811                                                      &pbd_e2_parsing_data,
2812                                                      xmit_type);
2813                 if (IS_MF_SI(bp)) {
2814                         /*
2815                          * fill in the MAC addresses in the PBD - for local
2816                          * switching
2817                          */
2818                         bnx2x_set_fw_mac_addr(&pbd_e2->src_mac_addr_hi,
2819                                               &pbd_e2->src_mac_addr_mid,
2820                                               &pbd_e2->src_mac_addr_lo,
2821                                               eth->h_source);
2822                         bnx2x_set_fw_mac_addr(&pbd_e2->dst_mac_addr_hi,
2823                                               &pbd_e2->dst_mac_addr_mid,
2824                                               &pbd_e2->dst_mac_addr_lo,
2825                                               eth->h_dest);
2826                 }
2827         } else {
2828                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
2829                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
2830                 /* Set PBD in checksum offload case */
2831                 if (xmit_type & XMIT_CSUM)
2832                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
2833
2834         }
2835
2836         /* Setup the data pointer of the first BD of the packet */
2837         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2838         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2839         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
2840         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2841         pkt_size = tx_start_bd->nbytes;
2842
2843         DP(NETIF_MSG_TX_QUEUED,
2844            "first bd @%p  addr (%x:%x)  nbd %d  nbytes %d  flags %x  vlan %x\n",
2845            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2846            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2847            tx_start_bd->bd_flags.as_bitfield,
2848            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
2849
2850         if (xmit_type & XMIT_GSO) {
2851
2852                 DP(NETIF_MSG_TX_QUEUED,
2853                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
2854                    skb->len, hlen, skb_headlen(skb),
2855                    skb_shinfo(skb)->gso_size);
2856
2857                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2858
2859                 if (unlikely(skb_headlen(skb) > hlen))
2860                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
2861                                                  &tx_start_bd, hlen,
2862                                                  bd_prod, ++nbd);
2863                 if (!CHIP_IS_E1x(bp))
2864                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
2865                                              xmit_type);
2866                 else
2867                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
2868         }
2869
2870         /* Set the PBD's parsing_data field if not zero
2871          * (for the chips newer than 57711).
2872          */
2873         if (pbd_e2_parsing_data)
2874                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
2875
2876         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2877
2878         /* Handle fragmented skb */
2879         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2880                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2881
2882                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
2883                                            skb_frag_size(frag), DMA_TO_DEVICE);
2884                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2885                         unsigned int pkts_compl = 0, bytes_compl = 0;
2886
2887                         DP(NETIF_MSG_TX_QUEUED,
2888                            "Unable to map page - dropping packet...\n");
2889
2890                         /* we need unmap all buffers already mapped
2891                          * for this SKB;
2892                          * first_bd->nbd need to be properly updated
2893                          * before call to bnx2x_free_tx_pkt
2894                          */
2895                         first_bd->nbd = cpu_to_le16(nbd);
2896                         bnx2x_free_tx_pkt(bp, txdata,
2897                                           TX_BD(txdata->tx_pkt_prod),
2898                                           &pkts_compl, &bytes_compl);
2899                         return NETDEV_TX_OK;
2900                 }
2901
2902                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2903                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2904                 if (total_pkt_bd == NULL)
2905                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2906
2907                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2908                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2909                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
2910                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
2911                 nbd++;
2912
2913                 DP(NETIF_MSG_TX_QUEUED,
2914                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
2915                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2916                    le16_to_cpu(tx_data_bd->nbytes));
2917         }
2918
2919         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2920
2921         /* update with actual num BDs */
2922         first_bd->nbd = cpu_to_le16(nbd);
2923
2924         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2925
2926         /* now send a tx doorbell, counting the next BD
2927          * if the packet contains or ends with it
2928          */
2929         if (TX_BD_POFF(bd_prod) < nbd)
2930                 nbd++;
2931
2932         /* total_pkt_bytes should be set on the first data BD if
2933          * it's not an LSO packet and there is more than one
2934          * data BD. In this case pkt_size is limited by an MTU value.
2935          * However we prefer to set it for an LSO packet (while we don't
2936          * have to) in order to save some CPU cycles in a none-LSO
2937          * case, when we much more care about them.
2938          */
2939         if (total_pkt_bd != NULL)
2940                 total_pkt_bd->total_pkt_bytes = pkt_size;
2941
2942         if (pbd_e1x)
2943                 DP(NETIF_MSG_TX_QUEUED,
2944                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
2945                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
2946                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
2947                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
2948                     le16_to_cpu(pbd_e1x->total_hlen_w));
2949         if (pbd_e2)
2950                 DP(NETIF_MSG_TX_QUEUED,
2951                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
2952                    pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
2953                    pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
2954                    pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
2955                    pbd_e2->parsing_data);
2956         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
2957
2958         netdev_tx_sent_queue(txq, skb->len);
2959
2960         txdata->tx_pkt_prod++;
2961         /*
2962          * Make sure that the BD data is updated before updating the producer
2963          * since FW might read the BD right after the producer is updated.
2964          * This is only applicable for weak-ordered memory model archs such
2965          * as IA-64. The following barrier is also mandatory since FW will
2966          * assumes packets must have BDs.
2967          */
2968         wmb();
2969
2970         txdata->tx_db.data.prod += nbd;
2971         barrier();
2972
2973         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
2974
2975         mmiowb();
2976
2977         txdata->tx_bd_prod += nbd;
2978
2979         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_SKB_FRAGS + 3)) {
2980                 netif_tx_stop_queue(txq);
2981
2982                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2983                  * ordering of set_bit() in netif_tx_stop_queue() and read of
2984                  * fp->bd_tx_cons */
2985                 smp_mb();
2986
2987                 fp->eth_q_stats.driver_xoff++;
2988                 if (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3)
2989                         netif_tx_wake_queue(txq);
2990         }
2991         txdata->tx_pkt++;
2992
2993         return NETDEV_TX_OK;
2994 }
2995
2996 /**
2997  * bnx2x_setup_tc - routine to configure net_device for multi tc
2998  *
2999  * @netdev: net device to configure
3000  * @tc: number of traffic classes to enable
3001  *
3002  * callback connected to the ndo_setup_tc function pointer
3003  */
3004 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
3005 {
3006         int cos, prio, count, offset;
3007         struct bnx2x *bp = netdev_priv(dev);
3008
3009         /* setup tc must be called under rtnl lock */
3010         ASSERT_RTNL();
3011
3012         /* no traffic classes requested. aborting */
3013         if (!num_tc) {
3014                 netdev_reset_tc(dev);
3015                 return 0;
3016         }
3017
3018         /* requested to support too many traffic classes */
3019         if (num_tc > bp->max_cos) {
3020                 BNX2X_ERR("support for too many traffic classes requested: %d. max supported is %d\n",
3021                           num_tc, bp->max_cos);
3022                 return -EINVAL;
3023         }
3024
3025         /* declare amount of supported traffic classes */
3026         if (netdev_set_num_tc(dev, num_tc)) {
3027                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
3028                 return -EINVAL;
3029         }
3030
3031         /* configure priority to traffic class mapping */
3032         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
3033                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
3034                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3035                    "mapping priority %d to tc %d\n",
3036                    prio, bp->prio_to_cos[prio]);
3037         }
3038
3039
3040         /* Use this configuration to diffrentiate tc0 from other COSes
3041            This can be used for ets or pfc, and save the effort of setting
3042            up a multio class queue disc or negotiating DCBX with a switch
3043         netdev_set_prio_tc_map(dev, 0, 0);
3044         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
3045         for (prio = 1; prio < 16; prio++) {
3046                 netdev_set_prio_tc_map(dev, prio, 1);
3047                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
3048         } */
3049
3050         /* configure traffic class to transmission queue mapping */
3051         for (cos = 0; cos < bp->max_cos; cos++) {
3052                 count = BNX2X_NUM_ETH_QUEUES(bp);
3053                 offset = cos * MAX_TXQS_PER_COS;
3054                 netdev_set_tc_queue(dev, cos, count, offset);
3055                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3056                    "mapping tc %d to offset %d count %d\n",
3057                    cos, offset, count);
3058         }
3059
3060         return 0;
3061 }
3062
3063 /* called with rtnl_lock */
3064 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
3065 {
3066         struct sockaddr *addr = p;
3067         struct bnx2x *bp = netdev_priv(dev);
3068         int rc = 0;
3069
3070         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
3071                 BNX2X_ERR("Requested MAC address is not valid\n");
3072                 return -EINVAL;
3073         }
3074
3075 #ifdef BCM_CNIC
3076         if (IS_MF_STORAGE_SD(bp) && !is_zero_ether_addr(addr->sa_data)) {
3077                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
3078                 return -EINVAL;
3079         }
3080 #endif
3081
3082         if (netif_running(dev))  {
3083                 rc = bnx2x_set_eth_mac(bp, false);
3084                 if (rc)
3085                         return rc;
3086         }
3087
3088         dev->addr_assign_type &= ~NET_ADDR_RANDOM;
3089         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
3090
3091         if (netif_running(dev))
3092                 rc = bnx2x_set_eth_mac(bp, true);
3093
3094         return rc;
3095 }
3096
3097 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
3098 {
3099         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
3100         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
3101         u8 cos;
3102
3103         /* Common */
3104 #ifdef BCM_CNIC
3105         if (IS_FCOE_IDX(fp_index)) {
3106                 memset(sb, 0, sizeof(union host_hc_status_block));
3107                 fp->status_blk_mapping = 0;
3108
3109         } else {
3110 #endif
3111                 /* status blocks */
3112                 if (!CHIP_IS_E1x(bp))
3113                         BNX2X_PCI_FREE(sb->e2_sb,
3114                                        bnx2x_fp(bp, fp_index,
3115                                                 status_blk_mapping),
3116                                        sizeof(struct host_hc_status_block_e2));
3117                 else
3118                         BNX2X_PCI_FREE(sb->e1x_sb,
3119                                        bnx2x_fp(bp, fp_index,
3120                                                 status_blk_mapping),
3121                                        sizeof(struct host_hc_status_block_e1x));
3122 #ifdef BCM_CNIC
3123         }
3124 #endif
3125         /* Rx */
3126         if (!skip_rx_queue(bp, fp_index)) {
3127                 bnx2x_free_rx_bds(fp);
3128
3129                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3130                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
3131                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
3132                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
3133                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
3134
3135                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
3136                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
3137                                sizeof(struct eth_fast_path_rx_cqe) *
3138                                NUM_RCQ_BD);
3139
3140                 /* SGE ring */
3141                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
3142                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
3143                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
3144                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3145         }
3146
3147         /* Tx */
3148         if (!skip_tx_queue(bp, fp_index)) {
3149                 /* fastpath tx rings: tx_buf tx_desc */
3150                 for_each_cos_in_tx_queue(fp, cos) {
3151                         struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3152
3153                         DP(NETIF_MSG_IFDOWN,
3154                            "freeing tx memory of fp %d cos %d cid %d\n",
3155                            fp_index, cos, txdata->cid);
3156
3157                         BNX2X_FREE(txdata->tx_buf_ring);
3158                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
3159                                 txdata->tx_desc_mapping,
3160                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3161                 }
3162         }
3163         /* end of fastpath */
3164 }
3165
3166 void bnx2x_free_fp_mem(struct bnx2x *bp)
3167 {
3168         int i;
3169         for_each_queue(bp, i)
3170                 bnx2x_free_fp_mem_at(bp, i);
3171 }
3172
3173 static inline void set_sb_shortcuts(struct bnx2x *bp, int index)
3174 {
3175         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
3176         if (!CHIP_IS_E1x(bp)) {
3177                 bnx2x_fp(bp, index, sb_index_values) =
3178                         (__le16 *)status_blk.e2_sb->sb.index_values;
3179                 bnx2x_fp(bp, index, sb_running_index) =
3180                         (__le16 *)status_blk.e2_sb->sb.running_index;
3181         } else {
3182                 bnx2x_fp(bp, index, sb_index_values) =
3183                         (__le16 *)status_blk.e1x_sb->sb.index_values;
3184                 bnx2x_fp(bp, index, sb_running_index) =
3185                         (__le16 *)status_blk.e1x_sb->sb.running_index;
3186         }
3187 }
3188
3189 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
3190 {
3191         union host_hc_status_block *sb;
3192         struct bnx2x_fastpath *fp = &bp->fp[index];
3193         int ring_size = 0;
3194         u8 cos;
3195         int rx_ring_size = 0;
3196
3197 #ifdef BCM_CNIC
3198         if (!bp->rx_ring_size && IS_MF_STORAGE_SD(bp)) {
3199                 rx_ring_size = MIN_RX_SIZE_NONTPA;
3200                 bp->rx_ring_size = rx_ring_size;
3201         } else
3202 #endif
3203         if (!bp->rx_ring_size) {
3204                 u32 cfg = SHMEM_RD(bp,
3205                              dev_info.port_hw_config[BP_PORT(bp)].default_cfg);
3206
3207                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
3208
3209                 /* Dercease ring size for 1G functions */
3210                 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
3211                     PORT_HW_CFG_NET_SERDES_IF_SGMII)
3212                         rx_ring_size /= 10;
3213
3214                 /* allocate at least number of buffers required by FW */
3215                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
3216                                      MIN_RX_SIZE_TPA, rx_ring_size);
3217
3218                 bp->rx_ring_size = rx_ring_size;
3219         } else /* if rx_ring_size specified - use it */
3220                 rx_ring_size = bp->rx_ring_size;
3221
3222         /* Common */
3223         sb = &bnx2x_fp(bp, index, status_blk);
3224 #ifdef BCM_CNIC
3225         if (!IS_FCOE_IDX(index)) {
3226 #endif
3227                 /* status blocks */
3228                 if (!CHIP_IS_E1x(bp))
3229                         BNX2X_PCI_ALLOC(sb->e2_sb,
3230                                 &bnx2x_fp(bp, index, status_blk_mapping),
3231                                 sizeof(struct host_hc_status_block_e2));
3232                 else
3233                         BNX2X_PCI_ALLOC(sb->e1x_sb,
3234                                 &bnx2x_fp(bp, index, status_blk_mapping),
3235                             sizeof(struct host_hc_status_block_e1x));
3236 #ifdef BCM_CNIC
3237         }
3238 #endif
3239
3240         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
3241          * set shortcuts for it.
3242          */
3243         if (!IS_FCOE_IDX(index))
3244                 set_sb_shortcuts(bp, index);
3245
3246         /* Tx */
3247         if (!skip_tx_queue(bp, index)) {
3248                 /* fastpath tx rings: tx_buf tx_desc */
3249                 for_each_cos_in_tx_queue(fp, cos) {
3250                         struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3251
3252                         DP(NETIF_MSG_IFUP,
3253                            "allocating tx memory of fp %d cos %d\n",
3254                            index, cos);
3255
3256                         BNX2X_ALLOC(txdata->tx_buf_ring,
3257                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
3258                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
3259                                 &txdata->tx_desc_mapping,
3260                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3261                 }
3262         }
3263
3264         /* Rx */
3265         if (!skip_rx_queue(bp, index)) {
3266                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3267                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
3268                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
3269                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
3270                                 &bnx2x_fp(bp, index, rx_desc_mapping),
3271                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3272
3273                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
3274                                 &bnx2x_fp(bp, index, rx_comp_mapping),
3275                                 sizeof(struct eth_fast_path_rx_cqe) *
3276                                 NUM_RCQ_BD);
3277
3278                 /* SGE ring */
3279                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
3280                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
3281                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
3282                                 &bnx2x_fp(bp, index, rx_sge_mapping),
3283                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3284                 /* RX BD ring */
3285                 bnx2x_set_next_page_rx_bd(fp);
3286
3287                 /* CQ ring */
3288                 bnx2x_set_next_page_rx_cq(fp);
3289
3290                 /* BDs */
3291                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
3292                 if (ring_size < rx_ring_size)
3293                         goto alloc_mem_err;
3294         }
3295
3296         return 0;
3297
3298 /* handles low memory cases */
3299 alloc_mem_err:
3300         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
3301                                                 index, ring_size);
3302         /* FW will drop all packets if queue is not big enough,
3303          * In these cases we disable the queue
3304          * Min size is different for OOO, TPA and non-TPA queues
3305          */
3306         if (ring_size < (fp->disable_tpa ?
3307                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
3308                         /* release memory allocated for this queue */
3309                         bnx2x_free_fp_mem_at(bp, index);
3310                         return -ENOMEM;
3311         }
3312         return 0;
3313 }
3314
3315 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
3316 {
3317         int i;
3318
3319         /**
3320          * 1. Allocate FP for leading - fatal if error
3321          * 2. {CNIC} Allocate FCoE FP - fatal if error
3322          * 3. {CNIC} Allocate OOO + FWD - disable OOO if error
3323          * 4. Allocate RSS - fix number of queues if error
3324          */
3325
3326         /* leading */
3327         if (bnx2x_alloc_fp_mem_at(bp, 0))
3328                 return -ENOMEM;
3329
3330 #ifdef BCM_CNIC
3331         if (!NO_FCOE(bp))
3332                 /* FCoE */
3333                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX))
3334                         /* we will fail load process instead of mark
3335                          * NO_FCOE_FLAG
3336                          */
3337                         return -ENOMEM;
3338 #endif
3339
3340         /* RSS */
3341         for_each_nondefault_eth_queue(bp, i)
3342                 if (bnx2x_alloc_fp_mem_at(bp, i))
3343                         break;
3344
3345         /* handle memory failures */
3346         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
3347                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
3348
3349                 WARN_ON(delta < 0);
3350 #ifdef BCM_CNIC
3351                 /**
3352                  * move non eth FPs next to last eth FP
3353                  * must be done in that order
3354                  * FCOE_IDX < FWD_IDX < OOO_IDX
3355                  */
3356
3357                 /* move FCoE fp even NO_FCOE_FLAG is on */
3358                 bnx2x_move_fp(bp, FCOE_IDX, FCOE_IDX - delta);
3359 #endif
3360                 bp->num_queues -= delta;
3361                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
3362                           bp->num_queues + delta, bp->num_queues);
3363         }
3364
3365         return 0;
3366 }
3367
3368 void bnx2x_free_mem_bp(struct bnx2x *bp)
3369 {
3370         kfree(bp->fp);
3371         kfree(bp->msix_table);
3372         kfree(bp->ilt);
3373 }
3374
3375 int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
3376 {
3377         struct bnx2x_fastpath *fp;
3378         struct msix_entry *tbl;
3379         struct bnx2x_ilt *ilt;
3380         int msix_table_size = 0;
3381
3382         /*
3383          * The biggest MSI-X table we might need is as a maximum number of fast
3384          * path IGU SBs plus default SB (for PF).
3385          */
3386         msix_table_size = bp->igu_sb_cnt + 1;
3387
3388         /* fp array: RSS plus CNIC related L2 queues */
3389         fp = kcalloc(BNX2X_MAX_RSS_COUNT(bp) + NON_ETH_CONTEXT_USE,
3390                      sizeof(*fp), GFP_KERNEL);
3391         if (!fp)
3392                 goto alloc_err;
3393         bp->fp = fp;
3394
3395         /* msix table */
3396         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
3397         if (!tbl)
3398                 goto alloc_err;
3399         bp->msix_table = tbl;
3400
3401         /* ilt */
3402         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
3403         if (!ilt)
3404                 goto alloc_err;
3405         bp->ilt = ilt;
3406
3407         return 0;
3408 alloc_err:
3409         bnx2x_free_mem_bp(bp);
3410         return -ENOMEM;
3411
3412 }
3413
3414 int bnx2x_reload_if_running(struct net_device *dev)
3415 {
3416         struct bnx2x *bp = netdev_priv(dev);
3417
3418         if (unlikely(!netif_running(dev)))
3419                 return 0;
3420
3421         bnx2x_nic_unload(bp, UNLOAD_NORMAL);
3422         return bnx2x_nic_load(bp, LOAD_NORMAL);
3423 }
3424
3425 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
3426 {
3427         u32 sel_phy_idx = 0;
3428         if (bp->link_params.num_phys <= 1)
3429                 return INT_PHY;
3430
3431         if (bp->link_vars.link_up) {
3432                 sel_phy_idx = EXT_PHY1;
3433                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
3434                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
3435                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
3436                         sel_phy_idx = EXT_PHY2;
3437         } else {
3438
3439                 switch (bnx2x_phy_selection(&bp->link_params)) {
3440                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
3441                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
3442                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
3443                        sel_phy_idx = EXT_PHY1;
3444                        break;
3445                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
3446                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
3447                        sel_phy_idx = EXT_PHY2;
3448                        break;
3449                 }
3450         }
3451
3452         return sel_phy_idx;
3453
3454 }
3455 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
3456 {
3457         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
3458         /*
3459          * The selected actived PHY is always after swapping (in case PHY
3460          * swapping is enabled). So when swapping is enabled, we need to reverse
3461          * the configuration
3462          */
3463
3464         if (bp->link_params.multi_phy_config &
3465             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
3466                 if (sel_phy_idx == EXT_PHY1)
3467                         sel_phy_idx = EXT_PHY2;
3468                 else if (sel_phy_idx == EXT_PHY2)
3469                         sel_phy_idx = EXT_PHY1;
3470         }
3471         return LINK_CONFIG_IDX(sel_phy_idx);
3472 }
3473
3474 #if defined(NETDEV_FCOE_WWNN) && defined(BCM_CNIC)
3475 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
3476 {
3477         struct bnx2x *bp = netdev_priv(dev);
3478         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
3479
3480         switch (type) {
3481         case NETDEV_FCOE_WWNN:
3482                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
3483                                 cp->fcoe_wwn_node_name_lo);
3484                 break;
3485         case NETDEV_FCOE_WWPN:
3486                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
3487                                 cp->fcoe_wwn_port_name_lo);
3488                 break;
3489         default:
3490                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
3491                 return -EINVAL;
3492         }
3493
3494         return 0;
3495 }
3496 #endif
3497
3498 /* called with rtnl_lock */
3499 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
3500 {
3501         struct bnx2x *bp = netdev_priv(dev);
3502
3503         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3504                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
3505                 return -EAGAIN;
3506         }
3507
3508         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
3509             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
3510                 BNX2X_ERR("Can't support requested MTU size\n");
3511                 return -EINVAL;
3512         }
3513
3514         /* This does not race with packet allocation
3515          * because the actual alloc size is
3516          * only updated as part of load
3517          */
3518         dev->mtu = new_mtu;
3519
3520         bp->gro_check = bnx2x_need_gro_check(new_mtu);
3521
3522         return bnx2x_reload_if_running(dev);
3523 }
3524
3525 netdev_features_t bnx2x_fix_features(struct net_device *dev,
3526                                      netdev_features_t features)
3527 {
3528         struct bnx2x *bp = netdev_priv(dev);
3529
3530         /* TPA requires Rx CSUM offloading */
3531         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
3532                 features &= ~NETIF_F_LRO;
3533                 features &= ~NETIF_F_GRO;
3534         }
3535
3536         return features;
3537 }
3538
3539 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
3540 {
3541         struct bnx2x *bp = netdev_priv(dev);
3542         u32 flags = bp->flags;
3543         bool bnx2x_reload = false;
3544
3545         if (features & NETIF_F_LRO)
3546                 flags |= TPA_ENABLE_FLAG;
3547         else
3548                 flags &= ~TPA_ENABLE_FLAG;
3549
3550         if (features & NETIF_F_GRO)
3551                 flags |= GRO_ENABLE_FLAG;
3552         else
3553                 flags &= ~GRO_ENABLE_FLAG;
3554
3555         if (features & NETIF_F_LOOPBACK) {
3556                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
3557                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
3558                         bnx2x_reload = true;
3559                 }
3560         } else {
3561                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
3562                         bp->link_params.loopback_mode = LOOPBACK_NONE;
3563                         bnx2x_reload = true;
3564                 }
3565         }
3566
3567         if (flags ^ bp->flags) {
3568                 bp->flags = flags;
3569                 bnx2x_reload = true;
3570         }
3571
3572         if (bnx2x_reload) {
3573                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
3574                         return bnx2x_reload_if_running(dev);
3575                 /* else: bnx2x_nic_load() will be called at end of recovery */
3576         }
3577
3578         return 0;
3579 }
3580
3581 void bnx2x_tx_timeout(struct net_device *dev)
3582 {
3583         struct bnx2x *bp = netdev_priv(dev);
3584
3585 #ifdef BNX2X_STOP_ON_ERROR
3586         if (!bp->panic)
3587                 bnx2x_panic();
3588 #endif
3589
3590         smp_mb__before_clear_bit();
3591         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
3592         smp_mb__after_clear_bit();
3593
3594         /* This allows the netif to be shutdown gracefully before resetting */
3595         schedule_delayed_work(&bp->sp_rtnl_task, 0);
3596 }
3597
3598 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
3599 {
3600         struct net_device *dev = pci_get_drvdata(pdev);
3601         struct bnx2x *bp;
3602
3603         if (!dev) {
3604                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3605                 return -ENODEV;
3606         }
3607         bp = netdev_priv(dev);
3608
3609         rtnl_lock();
3610
3611         pci_save_state(pdev);
3612
3613         if (!netif_running(dev)) {
3614                 rtnl_unlock();
3615                 return 0;
3616         }
3617
3618         netif_device_detach(dev);
3619
3620         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
3621
3622         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
3623
3624         rtnl_unlock();
3625
3626         return 0;
3627 }
3628
3629 int bnx2x_resume(struct pci_dev *pdev)
3630 {
3631         struct net_device *dev = pci_get_drvdata(pdev);
3632         struct bnx2x *bp;
3633         int rc;
3634
3635         if (!dev) {
3636                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3637                 return -ENODEV;
3638         }
3639         bp = netdev_priv(dev);
3640
3641         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3642                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
3643                 return -EAGAIN;
3644         }
3645
3646         rtnl_lock();
3647
3648         pci_restore_state(pdev);
3649
3650         if (!netif_running(dev)) {
3651                 rtnl_unlock();
3652                 return 0;
3653         }
3654
3655         bnx2x_set_power_state(bp, PCI_D0);
3656         netif_device_attach(dev);
3657
3658         rc = bnx2x_nic_load(bp, LOAD_OPEN);
3659
3660         rtnl_unlock();
3661
3662         return rc;
3663 }
3664
3665
3666 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
3667                               u32 cid)
3668 {
3669         /* ustorm cxt validation */
3670         cxt->ustorm_ag_context.cdu_usage =
3671                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3672                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
3673         /* xcontext validation */
3674         cxt->xstorm_ag_context.cdu_reserved =
3675                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3676                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
3677 }
3678
3679 static inline void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
3680                                              u8 fw_sb_id, u8 sb_index,
3681                                              u8 ticks)
3682 {
3683
3684         u32 addr = BAR_CSTRORM_INTMEM +
3685                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
3686         REG_WR8(bp, addr, ticks);
3687         DP(NETIF_MSG_IFUP,
3688            "port %x fw_sb_id %d sb_index %d ticks %d\n",
3689            port, fw_sb_id, sb_index, ticks);
3690 }
3691
3692 static inline void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
3693                                              u16 fw_sb_id, u8 sb_index,
3694                                              u8 disable)
3695 {
3696         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
3697         u32 addr = BAR_CSTRORM_INTMEM +
3698                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
3699         u16 flags = REG_RD16(bp, addr);
3700         /* clear and set */
3701         flags &= ~HC_INDEX_DATA_HC_ENABLED;
3702         flags |= enable_flag;
3703         REG_WR16(bp, addr, flags);
3704         DP(NETIF_MSG_IFUP,
3705            "port %x fw_sb_id %d sb_index %d disable %d\n",
3706            port, fw_sb_id, sb_index, disable);
3707 }
3708
3709 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
3710                                     u8 sb_index, u8 disable, u16 usec)
3711 {
3712         int port = BP_PORT(bp);
3713         u8 ticks = usec / BNX2X_BTR;
3714
3715         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
3716
3717         disable = disable ? 1 : (usec ? 0 : 1);
3718         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
3719 }