drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2008 NetEffect, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static u32 crit_err_count;
  46 u32 int_mod_timer_init;
  47 u32 int_mod_cq_depth_256;
  48 u32 int_mod_cq_depth_128;
  49 u32 int_mod_cq_depth_32;
  50 u32 int_mod_cq_depth_24;
  51 u32 int_mod_cq_depth_16;
  52 u32 int_mod_cq_depth_4;
  53 u32 int_mod_cq_depth_1;
  54
  55 #include "nes_cm.h"
  56
  57 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  58 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  59 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  60                            u8 OneG_Mode);
  61 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  62 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  63 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  64 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  65                                    struct nes_hw_aeqe *aeqe);
  66 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  67 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  68
  69 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  70 static unsigned char *nes_iwarp_state_str[] = {
  71         "Non-Existant",
  72         "Idle",
  73         "RTS",
  74         "Closing",
  75         "RSVD1",
  76         "Terminate",
  77         "Error",
  78         "RSVD2",
  79 };
  80
  81 static unsigned char *nes_tcp_state_str[] = {
  82         "Non-Existant",
  83         "Closed",
  84         "Listen",
  85         "SYN Sent",
  86         "SYN Rcvd",
  87         "Established",
  88         "Close Wait",
  89         "FIN Wait 1",
  90         "Closing",
  91         "Last Ack",
  92         "FIN Wait 2",
  93         "Time Wait",
  94         "RSVD1",
  95         "RSVD2",
  96         "RSVD3",
  97         "RSVD4",
  98 };
  99 #endif
 100
 101
 102 /**
 103  * nes_nic_init_timer_defaults
 104  */
 105 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 106 {
 107         unsigned long flags;
 108         struct nes_adapter *nesadapter = nesdev->nesadapter;
 109         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 110
 111         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 112
 113         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 114         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 115         if (jumbomode) {
 116                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 117                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 118                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 119         } else {
 120                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 121                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 122                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 123         }
 124
 125         /* todo use netdev->mtu to set thresholds */
 126         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 127 }
 128
 129
 130 /**
 131  * nes_nic_init_timer
 132  */
 133 static void  nes_nic_init_timer(struct nes_device *nesdev)
 134 {
 135         unsigned long flags;
 136         struct nes_adapter *nesadapter = nesdev->nesadapter;
 137         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 138
 139         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 140
 141         if (shared_timer->timer_in_use_old == 0) {
 142                 nesdev->deepcq_count = 0;
 143                 shared_timer->timer_direction_upward = 0;
 144                 shared_timer->timer_direction_downward = 0;
 145                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 146                 shared_timer->timer_in_use_old = 0;
 147
 148         }
 149         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 150                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 151                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 152                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 153         }
 154         /* todo use netdev->mtu to set thresholds */
 155         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 156 }
 157
 158
 159 /**
 160  * nes_nic_tune_timer
 161  */
 162 static void nes_nic_tune_timer(struct nes_device *nesdev)
 163 {
 164         unsigned long flags;
 165         struct nes_adapter *nesadapter = nesdev->nesadapter;
 166         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 167         u16 cq_count = nesdev->currcq_count;
 168
 169         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 170
 171         if (shared_timer->cq_count_old <= cq_count)
 172                 shared_timer->cq_direction_downward = 0;
 173         else
 174                 shared_timer->cq_direction_downward++;
 175         shared_timer->cq_count_old = cq_count;
 176         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 177                 if (cq_count <= shared_timer->threshold_low &&
 178                     shared_timer->threshold_low > 4) {
 179                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 180                         shared_timer->cq_direction_downward=0;
 181                         nesdev->currcq_count = 0;
 182                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 183                         return;
 184                 }
 185         }
 186
 187         if (cq_count > 1) {
 188                 nesdev->deepcq_count += cq_count;
 189                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 190                         shared_timer->timer_direction_upward++;
 191                         shared_timer->timer_direction_downward = 0;
 192                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 193                         shared_timer->timer_direction_upward = 0;
 194                         shared_timer->timer_direction_downward = 0;
 195                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 196                         shared_timer->timer_direction_downward++;
 197                         shared_timer->timer_direction_upward = 0;
 198                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 199                         shared_timer->timer_in_use -= 2;
 200                         shared_timer->timer_direction_upward = 0;
 201                         shared_timer->timer_direction_downward++;
 202                 } else {
 203                         shared_timer->timer_in_use -= 4;
 204                         shared_timer->timer_direction_upward = 0;
 205                         shared_timer->timer_direction_downward++;
 206                 }
 207
 208                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 209                         shared_timer->timer_in_use += 3;
 210                         shared_timer->timer_direction_upward = 0;
 211                         shared_timer->timer_direction_downward = 0;
 212                 }
 213                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 214                         shared_timer->timer_in_use -= 4 ;
 215                         shared_timer->timer_direction_downward = 0;
 216                         shared_timer->timer_direction_upward = 0;
 217                 }
 218         }
 219
 220         /* boundary checking */
 221         if (shared_timer->timer_in_use > NES_NIC_FAST_TIMER_HIGH)
 222                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_HIGH;
 223         else if (shared_timer->timer_in_use < NES_NIC_FAST_TIMER_LOW) {
 224                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_LOW;
 225         }
 226
 227         nesdev->currcq_count = 0;
 228
 229         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 230 }
 231
 232
 233 /**
 234  * nes_init_adapter - initialize adapter
 235  */
 236 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 237         struct nes_adapter *nesadapter = NULL;
 238         unsigned long num_pds;
 239         u32 u32temp;
 240         u32 port_count;
 241         u16 max_rq_wrs;
 242         u16 max_sq_wrs;
 243         u32 max_mr;
 244         u32 max_256pbl;
 245         u32 max_4kpbl;
 246         u32 max_qp;
 247         u32 max_irrq;
 248         u32 max_cq;
 249         u32 hte_index_mask;
 250         u32 adapter_size;
 251         u32 arp_table_size;
 252         u16 vendor_id;
 253         u8  OneG_Mode;
 254         u8  func_index;
 255
 256         /* search the list of existing adapters */
 257         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 258                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 259                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 260                                 nesdev->pcidev->devfn,
 261                                 PCI_SLOT(nesadapter->devfn),
 262                                 nesadapter->bus_number,
 263                                 PCI_SLOT(nesdev->pcidev->devfn),
 264                                 nesdev->pcidev->bus->number );
 265                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 266                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 267                         nesadapter->ref_count++;
 268                         return nesadapter;
 269                 }
 270         }
 271
 272         /* no adapter found */
 273         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 274         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 275                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 276                                 hw_rev);
 277                 return NULL;
 278         }
 279
 280         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 281                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 282                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 283                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 284                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 285
 286         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 287
 288
 289         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 290                 return NULL;
 291         if (nes_init_serdes(nesdev, hw_rev, port_count, OneG_Mode))
 292                 return NULL;
 293         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 294
 295         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 296         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 297
 298         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 299         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 300                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 301                                 max_qp, u32temp);
 302                 max_qp = (u32)1 << (u32temp & 0x001f);
 303         }
 304
 305         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 306         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 307                         max_qp, hte_index_mask);
 308
 309         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 310
 311         max_irrq = 1 << (u32temp & 0x001f);
 312
 313         if (max_qp > max_irrq) {
 314                 max_qp = max_irrq;
 315                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 316                                 max_qp);
 317         }
 318
 319         /* there should be no reason to allocate more pds than qps */
 320         if (num_pds > max_qp)
 321                 num_pds = max_qp;
 322
 323         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 324         max_mr = (u32)8192 << (u32temp & 0x7);
 325
 326         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 327         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 328         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 329         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 330
 331         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 332         arp_table_size = 1 << u32temp;
 333
 334         adapter_size = (sizeof(struct nes_adapter) +
 335                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 336         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 337         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 338         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 339         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 340         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 341         adapter_size += sizeof(struct nes_qp **) * max_qp;
 342
 343         /* allocate a new adapter struct */
 344         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 345         if (nesadapter == NULL) {
 346                 return NULL;
 347         }
 348
 349         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 350                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 351
 352         /* populate the new nesadapter */
 353         nesadapter->devfn = nesdev->pcidev->devfn;
 354         nesadapter->bus_number = nesdev->pcidev->bus->number;
 355         nesadapter->ref_count = 1;
 356         nesadapter->timer_int_req = 0xffff0000;
 357         nesadapter->OneG_Mode = OneG_Mode;
 358         nesadapter->doorbell_start = nesdev->doorbell_region;
 359
 360         /* nesadapter->tick_delta = clk_divisor; */
 361         nesadapter->hw_rev = hw_rev;
 362         nesadapter->port_count = port_count;
 363
 364         nesadapter->max_qp = max_qp;
 365         nesadapter->hte_index_mask = hte_index_mask;
 366         nesadapter->max_irrq = max_irrq;
 367         nesadapter->max_mr = max_mr;
 368         nesadapter->max_256pbl = max_256pbl - 1;
 369         nesadapter->max_4kpbl = max_4kpbl - 1;
 370         nesadapter->max_cq = max_cq;
 371         nesadapter->free_256pbl = max_256pbl - 1;
 372         nesadapter->free_4kpbl = max_4kpbl - 1;
 373         nesadapter->max_pd = num_pds;
 374         nesadapter->arp_table_size = arp_table_size;
 375
 376         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 377         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 378                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 379                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 380                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 381         } else {
 382                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 383                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 384                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 385                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 386         }
 387         /* Setup and enable the periodic timer */
 388         if (nesadapter->et_rx_coalesce_usecs_irq)
 389                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 390                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 391         else
 392                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 393
 394         nesadapter->base_pd = 1;
 395
 396         nesadapter->device_cap_flags =
 397                 IB_DEVICE_ZERO_STAG | IB_DEVICE_MEM_WINDOW;
 398
 399         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 400                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 401         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 402         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 403         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 404         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 405         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 406
 407
 408         /* mark the usual suspect QPs and CQs as in use */
 409         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 410                 set_bit(u32temp, nesadapter->allocated_qps);
 411                 set_bit(u32temp, nesadapter->allocated_cqs);
 412         }
 413
 414         for (u32temp = 0; u32temp < 20; u32temp++)
 415                 set_bit(u32temp, nesadapter->allocated_pds);
 416         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 417
 418         max_rq_wrs = ((u32temp >> 8) & 3);
 419         switch (max_rq_wrs) {
 420                 case 0:
 421                         max_rq_wrs = 4;
 422                         break;
 423                 case 1:
 424                         max_rq_wrs = 16;
 425                         break;
 426                 case 2:
 427                         max_rq_wrs = 32;
 428                         break;
 429                 case 3:
 430                         max_rq_wrs = 512;
 431                         break;
 432         }
 433
 434         max_sq_wrs = (u32temp & 3);
 435         switch (max_sq_wrs) {
 436                 case 0:
 437                         max_sq_wrs = 4;
 438                         break;
 439                 case 1:
 440                         max_sq_wrs = 16;
 441                         break;
 442                 case 2:
 443                         max_sq_wrs = 32;
 444                         break;
 445                 case 3:
 446                         max_sq_wrs = 512;
 447                         break;
 448         }
 449         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 450         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 451
 452         nesadapter->max_sge = 4;
 453         nesadapter->max_cqe = 32767;
 454
 455         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 456                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 457                 kfree(nesadapter);
 458                 return NULL;
 459         }
 460
 461         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 462         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 463                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 464
 465         /* setup port configuration */
 466         if (nesadapter->port_count == 1) {
 467                 u32temp = 0x00000000;
 468                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 469                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 470                 else
 471                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 472         } else {
 473                 if (nesadapter->port_count == 2)
 474                         u32temp = 0x00000044;
 475                 else
 476                         u32temp = 0x000000e4;
 477                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 478         }
 479
 480         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT, u32temp);
 481         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 482                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 483
 484         spin_lock_init(&nesadapter->resource_lock);
 485         spin_lock_init(&nesadapter->phy_lock);
 486         spin_lock_init(&nesadapter->pbl_lock);
 487         spin_lock_init(&nesadapter->periodic_timer_lock);
 488
 489         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 490         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 491         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 492         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 493
 494         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 495                 u32 pcs_control_status0, pcs_control_status1;
 496                 u32 reset_value;
 497                 u32 i = 0;
 498                 u32 int_cnt = 0;
 499                 u32 ext_cnt = 0;
 500                 unsigned long flags;
 501                 u32 j = 0;
 502
 503                 pcs_control_status0 = nes_read_indexed(nesdev,
 504                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 505                 pcs_control_status1 = nes_read_indexed(nesdev,
 506                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 507
 508                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 509                         pcs_control_status0 = nes_read_indexed(nesdev,
 510                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 511                         pcs_control_status1 = nes_read_indexed(nesdev,
 512                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 513                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 514                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 515                                 int_cnt++;
 516                         msleep(1);
 517                 }
 518                 if (int_cnt > 1) {
 519                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 520                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
 521                         mh_detected++;
 522                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 523                         reset_value |= 0x0000003d;
 524                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 525
 526                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 527                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 528                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 529
 530                         pcs_control_status0 = nes_read_indexed(nesdev,
 531                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 532                         pcs_control_status1 = nes_read_indexed(nesdev,
 533                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 534
 535                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 536                                 pcs_control_status0 = nes_read_indexed(nesdev,
 537                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 538                                 pcs_control_status1 = nes_read_indexed(nesdev,
 539                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 540                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 541                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 542                                         if (++ext_cnt > int_cnt) {
 543                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 544                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 545                                                                 0x0000F0C8);
 546                                                 mh_detected++;
 547                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 548                                                 reset_value |= 0x0000003d;
 549                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 550
 551                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 552                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 553                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 554                                                 break;
 555                                         }
 556                                 }
 557                                 msleep(1);
 558                         }
 559                 }
 560         }
 561
 562         if (nesadapter->hw_rev == NE020_REV) {
 563                 init_timer(&nesadapter->mh_timer);
 564                 nesadapter->mh_timer.function = nes_mh_fix;
 565                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 566                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 567                 add_timer(&nesadapter->mh_timer);
 568         } else {
 569                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 570         }
 571
 572         init_timer(&nesadapter->lc_timer);
 573         nesadapter->lc_timer.function = nes_clc;
 574         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 575         nesadapter->lc_timer.data = (unsigned long)nesdev;
 576         add_timer(&nesadapter->lc_timer);
 577
 578         list_add_tail(&nesadapter->list, &nes_adapter_list);
 579
 580         for (func_index = 0; func_index < 8; func_index++) {
 581                 pci_bus_read_config_word(nesdev->pcidev->bus,
 582                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 583                                         func_index), 0, &vendor_id);
 584                 if (vendor_id == 0xffff)
 585                         break;
 586         }
 587         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 588                 func_index, pci_name(nesdev->pcidev));
 589         nesadapter->adapter_fcn_count = func_index;
 590
 591         return nesadapter;
 592 }
 593
 594
 595 /**
 596  * nes_reset_adapter_ne020
 597  */
 598 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 599 {
 600         u32 port_count;
 601         u32 u32temp;
 602         u32 i;
 603
 604         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 605         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 606         /* TODO: assuming that both SERDES are set the same for now */
 607         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 608         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 609                         u32temp, port_count);
 610         if (*OneG_Mode)
 611                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 612         u32temp &= 0xff00ffc0;
 613         switch (port_count) {
 614                 case 1:
 615                         u32temp |= 0x00ee0000;
 616                         break;
 617                 case 2:
 618                         u32temp |= 0x00cc0000;
 619                         break;
 620                 case 4:
 621                         u32temp |= 0x00000000;
 622                         break;
 623                 default:
 624                         return 0;
 625                         break;
 626         }
 627
 628         /* check and do full reset if needed */
 629         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 630                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 631                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 632
 633                 i = 0;
 634                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 635                         mdelay(1);
 636                 if (i >= 10000) {
 637                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 638                         return 0;
 639                 }
 640
 641                 i = 0;
 642                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 643                         mdelay(1);
 644                 if (i >= 10000) {
 645                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 646                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 647                         return 0;
 648                 }
 649         }
 650
 651         /* port reset */
 652         switch (port_count) {
 653                 case 1:
 654                         u32temp |= 0x00ee0010;
 655                         break;
 656                 case 2:
 657                         u32temp |= 0x00cc0030;
 658                         break;
 659                 case 4:
 660                         u32temp |= 0x00000030;
 661                         break;
 662         }
 663
 664         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 665         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 666
 667         i = 0;
 668         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 669                 mdelay(1);
 670         if (i >= 10000) {
 671                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 672                 return 0;
 673         }
 674
 675         /* serdes 0 */
 676         i = 0;
 677         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 678                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 679                 mdelay(1);
 680         if (i >= 5000) {
 681                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 682                 return 0;
 683         }
 684
 685         /* serdes 1 */
 686         if (port_count > 1) {
 687                 i = 0;
 688                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 689                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 690                         mdelay(1);
 691                 if (i >= 5000) {
 692                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 693                         return 0;
 694                 }
 695         }
 696
 697         return port_count;
 698 }
 699
 700
 701 /**
 702  * nes_init_serdes
 703  */
 704 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 705                            u8 OneG_Mode)
 706 {
 707         int i;
 708         u32 u32temp;
 709
 710         if (hw_rev != NE020_REV) {
 711                 /* init serdes 0 */
 712
 713                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 714                 if (!OneG_Mode)
 715                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 716                 if (port_count > 1) {
 717                         /* init serdes 1 */
 718                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 719                         if (!OneG_Mode)
 720                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 721                         }
 722         } else {
 723                 /* init serdes 0 */
 724                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 725                 i = 0;
 726                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 727                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 728                         mdelay(1);
 729                 if (i >= 5000) {
 730                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 731                         return 1;
 732                 }
 733                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 734                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 735                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 736                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 737                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 738                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 739                 if (OneG_Mode)
 740                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 741                 else
 742                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 743
 744                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 745                 if (port_count > 1) {
 746                         /* init serdes 1 */
 747                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 748                         i = 0;
 749                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 750                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 751                                 mdelay(1);
 752                         if (i >= 5000) {
 753                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 754                                 /* return 1; */
 755                         }
 756                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 757                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 758                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 759                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 761                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 762                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 763                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 764                 }
 765         }
 766         return 0;
 767 }
 768
 769
 770 /**
 771  * nes_init_csr_ne020
 772  * Initialize registers for ne020 hardware
 773  */
 774 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 775 {
 776         u32 u32temp;
 777
 778         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 779
 780         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 781         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 782         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 783         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 784         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 785         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 786         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 787         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 788
 789         /* TODO: move these MAC register settings to NIC bringup */
 790         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 791         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 792         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 793         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 794         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 795         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 796         if (port_count > 1) {
 797                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 798                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 799                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 800                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 801                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 802                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 803                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 804         }
 805         if (port_count > 2) {
 806                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 807                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 808                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 809                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 810                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 811                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 812                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 813
 814                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 815                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 816                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 817                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 818                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 819                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 820                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 821         }
 822
 823         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 824         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 825         nes_write_indexed(nesdev, 0x00005004, 0x00020001);
 826         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 827         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 828         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 829         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 830         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 831
 832         /* TODO: move this to code, get from EEPROM */
 833         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 834         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 835         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 836
 837         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 838         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 839
 840         if (hw_rev != NE020_REV) {
 841                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 842                 u32temp |= 0x80000000;
 843                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 844                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 845                 u32temp &= 0x7fffffff;
 846                 u32temp |= 0x7fff0010;
 847                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 848         }
 849 }
 850
 851
 852 /**
 853  * nes_destroy_adapter - destroy the adapter structure
 854  */
 855 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 856 {
 857         struct nes_adapter *tmp_adapter;
 858
 859         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 860                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 861                                 tmp_adapter);
 862         }
 863
 864         nesadapter->ref_count--;
 865         if (!nesadapter->ref_count) {
 866                 if (nesadapter->hw_rev == NE020_REV) {
 867                         del_timer(&nesadapter->mh_timer);
 868                 }
 869                 del_timer(&nesadapter->lc_timer);
 870
 871                 list_del(&nesadapter->list);
 872                 kfree(nesadapter);
 873         }
 874 }
 875
 876
 877 /**
 878  * nes_init_cqp
 879  */
 880 int nes_init_cqp(struct nes_device *nesdev)
 881 {
 882         struct nes_adapter *nesadapter = nesdev->nesadapter;
 883         struct nes_hw_cqp_qp_context *cqp_qp_context;
 884         struct nes_hw_cqp_wqe *cqp_wqe;
 885         struct nes_hw_ceq *ceq;
 886         struct nes_hw_ceq *nic_ceq;
 887         struct nes_hw_aeq *aeq;
 888         void *vmem;
 889         dma_addr_t pmem;
 890         u32 count=0;
 891         u32 cqp_head;
 892         u64 u64temp;
 893         u32 u32temp;
 894
 895         /* allocate CQP memory */
 896         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 897         /* SQ is 512 byte aligned, others are 256 byte aligned */
 898         nesdev->cqp_mem_size = 512 +
 899                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 900                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 901                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 902                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 903                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 904                         sizeof(struct nes_hw_cqp_qp_context);
 905
 906         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 907                         &nesdev->cqp_pbase);
 908         if (!nesdev->cqp_vbase) {
 909                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 910                 return -ENOMEM;
 911         }
 912         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 913
 914         /* Allocate a twice the number of CQP requests as the SQ size */
 915         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 916                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 917         if (nesdev->nes_cqp_requests == NULL) {
 918                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 919                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 920                                 nesdev->cqp.sq_pbase);
 921                 return -ENOMEM;
 922         }
 923
 924         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 925                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 926
 927         spin_lock_init(&nesdev->cqp.lock);
 928         init_waitqueue_head(&nesdev->cqp.waitq);
 929
 930         /* Setup Various Structures */
 931         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 932                         ~(unsigned long)(512 - 1));
 933         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
 934                         ~(unsigned long long)(512 - 1));
 935
 936         nesdev->cqp.sq_vbase = vmem;
 937         nesdev->cqp.sq_pbase = pmem;
 938         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
 939         nesdev->cqp.sq_head = 0;
 940         nesdev->cqp.sq_tail = 0;
 941         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
 942
 943         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 944         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 945
 946         nesdev->ccq.cq_vbase = vmem;
 947         nesdev->ccq.cq_pbase = pmem;
 948         nesdev->ccq.cq_size = NES_CCQ_SIZE;
 949         nesdev->ccq.cq_head = 0;
 950         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
 951         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
 952
 953         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 954         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 955
 956         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
 957         ceq = &nesadapter->ceq[nesdev->ceq_index];
 958         ceq->ceq_vbase = vmem;
 959         ceq->ceq_pbase = pmem;
 960         ceq->ceq_size = NES_CCEQ_SIZE;
 961         ceq->ceq_head = 0;
 962
 963         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 964         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 965
 966         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
 967         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
 968         nic_ceq->ceq_vbase = vmem;
 969         nic_ceq->ceq_pbase = pmem;
 970         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
 971         nic_ceq->ceq_head = 0;
 972
 973         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 974         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 975
 976         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
 977         aeq->aeq_vbase = vmem;
 978         aeq->aeq_pbase = pmem;
 979         aeq->aeq_size = nesadapter->max_qp;
 980         aeq->aeq_head = 0;
 981
 982         /* Setup QP Context */
 983         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 984         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 985
 986         cqp_qp_context = vmem;
 987         cqp_qp_context->context_words[0] =
 988                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
 989         cqp_qp_context->context_words[1] = 0;
 990         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
 991         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
 992
 993
 994         /* Write the address to Create CQP */
 995         if ((sizeof(dma_addr_t) > 4)) {
 996                 nes_write_indexed(nesdev,
 997                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
 998                                 ((u64)pmem) >> 32);
 999         } else {
1000                 nes_write_indexed(nesdev,
1001                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1002         }
1003         nes_write_indexed(nesdev,
1004                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1005                         (u32)pmem);
1006
1007         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1008         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1009
1010         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1011                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1012                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1013         }
1014
1015         /* Write Create CCQ WQE */
1016         cqp_head = nesdev->cqp.sq_head++;
1017         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1018         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1019         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1020                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1021                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1022         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1023                             (nesdev->ccq.cq_number |
1024                              ((u32)nesdev->ceq_index << 16)));
1025         u64temp = (u64)nesdev->ccq.cq_pbase;
1026         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1027         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1028         u64temp = (unsigned long)&nesdev->ccq;
1029         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1030                         cpu_to_le32((u32)(u64temp >> 1));
1031         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1032                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1033         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1034
1035         /* Write Create CEQ WQE */
1036         cqp_head = nesdev->cqp.sq_head++;
1037         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1038         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1039         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1040                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1041         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1042         u64temp = (u64)ceq->ceq_pbase;
1043         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1044
1045         /* Write Create AEQ WQE */
1046         cqp_head = nesdev->cqp.sq_head++;
1047         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1048         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1049         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1050                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1051         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1052         u64temp = (u64)aeq->aeq_pbase;
1053         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1054
1055         /* Write Create NIC CEQ WQE */
1056         cqp_head = nesdev->cqp.sq_head++;
1057         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1058         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1059         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1060                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1061         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1062         u64temp = (u64)nic_ceq->ceq_pbase;
1063         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1064
1065         /* Poll until CCQP done */
1066         count = 0;
1067         do {
1068                 if (count++ > 1000) {
1069                         printk(KERN_ERR PFX "Error creating CQP\n");
1070                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1071                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1072                         return -1;
1073                 }
1074                 udelay(10);
1075         } while (!(nes_read_indexed(nesdev,
1076                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1077
1078         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1079                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1080
1081         u32temp = 0x04800000;
1082         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1083
1084         /* wait for the CCQ, CEQ, and AEQ to get created */
1085         count = 0;
1086         do {
1087                 if (count++ > 1000) {
1088                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1089                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1090                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1091                         return -1;
1092                 }
1093                 udelay(10);
1094         } while (((nes_read_indexed(nesdev,
1095                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1096
1097         /* dump the QP status value */
1098         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1099                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1100
1101         nesdev->cqp.sq_tail++;
1102
1103         return 0;
1104 }
1105
1106
1107 /**
1108  * nes_destroy_cqp
1109  */
1110 int nes_destroy_cqp(struct nes_device *nesdev)
1111 {
1112         struct nes_hw_cqp_wqe *cqp_wqe;
1113         u32 count = 0;
1114         u32 cqp_head;
1115         unsigned long flags;
1116
1117         do {
1118                 if (count++ > 1000)
1119                         break;
1120                 udelay(10);
1121         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1122
1123         /* Reset CCQ */
1124         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1125                         nesdev->ccq.cq_number);
1126
1127         /* Disable device interrupts */
1128         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1129
1130         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1131
1132         /* Destroy the AEQ */
1133         cqp_head = nesdev->cqp.sq_head++;
1134         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1135         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1136         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1137                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1138         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1139
1140         /* Destroy the NIC CEQ */
1141         cqp_head = nesdev->cqp.sq_head++;
1142         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1143         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1144         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1145                         ((u32)nesdev->nic_ceq_index << 8));
1146
1147         /* Destroy the CEQ */
1148         cqp_head = nesdev->cqp.sq_head++;
1149         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1150         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1151         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1152                         (nesdev->ceq_index << 8));
1153
1154         /* Destroy the CCQ */
1155         cqp_head = nesdev->cqp.sq_head++;
1156         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1157         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1158         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1159         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1160                         ((u32)nesdev->ceq_index << 16));
1161
1162         /* Destroy CQP */
1163         cqp_head = nesdev->cqp.sq_head++;
1164         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1165         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1166         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1167                         NES_CQP_QP_TYPE_CQP);
1168         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1169
1170         barrier();
1171         /* Ring doorbell (5 WQEs) */
1172         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1173
1174         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1175
1176         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1177         count = 0;
1178         do {
1179                 if (count++ > 1000) {
1180                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1181                                         PCI_FUNC(nesdev->pcidev->devfn));
1182                         break;
1183                 }
1184                 udelay(10);
1185         } while (((nes_read_indexed(nesdev,
1186                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1187
1188         /* dump the QP status value */
1189         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1190                         PCI_FUNC(nesdev->pcidev->devfn),
1191                         nes_read_indexed(nesdev,
1192                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1193
1194         kfree(nesdev->nes_cqp_requests);
1195
1196         /* Free the control structures */
1197         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1198                         nesdev->cqp.sq_pbase);
1199
1200         return 0;
1201 }
1202
1203
1204 /**
1205  * nes_init_phy
1206  */
1207 int nes_init_phy(struct nes_device *nesdev)
1208 {
1209         struct nes_adapter *nesadapter = nesdev->nesadapter;
1210         u32 counter = 0;
1211         u32 sds_common_control0;
1212         u32 mac_index = nesdev->mac_index;
1213         u32 tx_config = 0;
1214         u16 phy_data;
1215         u32 temp_phy_data = 0;
1216         u32 temp_phy_data2 = 0;
1217         u32 i = 0;
1218
1219         if ((nesadapter->OneG_Mode) &&
1220             (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
1221                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1222                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
1223                         printk(PFX "%s: Programming mdc config for 1G\n", __func__);
1224                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1225                         tx_config |= 0x04;
1226                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1227                 }
1228
1229                 nes_read_1G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index], &phy_data);
1230                 nes_debug(NES_DBG_PHY, "Phy data from register 1 phy address %u = 0x%X.\n",
1231                                 nesadapter->phy_index[mac_index], phy_data);
1232                 nes_write_1G_phy_reg(nesdev, 23, nesadapter->phy_index[mac_index], 0xb000);
1233
1234                 /* Reset the PHY */
1235                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], 0x8000);
1236                 udelay(100);
1237                 counter = 0;
1238                 do {
1239                         nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1240                         nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1241                         if (counter++ > 100) break;
1242                 } while (phy_data & 0x8000);
1243
1244                 /* Setting no phy loopback */
1245                 phy_data &= 0xbfff;
1246                 phy_data |= 0x1140;
1247                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index],  phy_data);
1248                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1249                 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1250
1251                 nes_read_1G_phy_reg(nesdev, 0x17, nesadapter->phy_index[mac_index], &phy_data);
1252                 nes_debug(NES_DBG_PHY, "Phy data from register 0x17 = 0x%X.\n", phy_data);
1253
1254                 nes_read_1G_phy_reg(nesdev, 0x1e, nesadapter->phy_index[mac_index], &phy_data);
1255                 nes_debug(NES_DBG_PHY, "Phy data from register 0x1e = 0x%X.\n", phy_data);
1256
1257                 /* Setting the interrupt mask */
1258                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1259                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1260                 nes_write_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], 0xffee);
1261
1262                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1263                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1264
1265                 /* turning on flow control */
1266                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1267                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1268                 nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1269                                 (phy_data & ~(0x03E0)) | 0xc00);
1270                 /* nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1271                                 phy_data | 0xc00); */
1272                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1273                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1274
1275                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1276                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1277                 /* Clear Half duplex */
1278                 nes_write_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index],
1279                                 phy_data & ~(0x0100));
1280                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1281                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1282
1283                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1284                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
1285         } else {
1286                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) ||
1287                     (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1288                         /* setup 10G MDIO operation */
1289                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1290                         tx_config |= 0x14;
1291                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1292                 }
1293                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1294                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1295
1296                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1297                         mdelay(10);
1298                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1299                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1300
1301                         /*
1302                          * if firmware is already running (like from a
1303                          * driver un-load/load, don't do anything.
1304                          */
1305                         if (temp_phy_data == temp_phy_data2) {
1306                                 /* configure QT2505 AMCC PHY */
1307                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0x0000, 0x8000);
1308                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0000);
1309                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc302, 0x0044);
1310                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc318, 0x0052);
1311                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc319, 0x0008);
1312                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc31a, 0x0098);
1313                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0026, 0x0E00);
1314                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0027, 0x0000);
1315                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0028, 0xA528);
1316
1317                                 /*
1318                                  * remove micro from reset; chip boots from ROM,
1319                                  * uploads EEPROM f/w image, uC executes f/w
1320                                  */
1321                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0002);
1322
1323                                 /*
1324                                  * wait for heart beat to start to
1325                                  * know loading is done
1326                                  */
1327                                 counter = 0;
1328                                 do {
1329                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1330                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1331                                         if (counter++ > 1000) {
1332                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from heartbeat check <this is bad!!!> \n");
1333                                                 break;
1334                                         }
1335                                         mdelay(100);
1336                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1337                                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1338                                 } while ((temp_phy_data2 == temp_phy_data));
1339
1340                                 /*
1341                                  * wait for tracking to start to know
1342                                  * f/w is good to go
1343                                  */
1344                                 counter = 0;
1345                                 do {
1346                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7fd);
1347                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1348                                         if (counter++ > 1000) {
1349                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from status check <this is bad!!!> \n");
1350                                                 break;
1351                                         }
1352                                         mdelay(1000);
1353                                         /*
1354                                          * nes_debug(NES_DBG_PHY, "AMCC PHY- phy_status not ready yet = 0x%02X\n",
1355                                          *                      temp_phy_data);
1356                                          */
1357                                 } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
1358
1359                                 /* set LOS Control invert RXLOSB_I_PADINV */
1360                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd003, 0x0000);
1361                                 /* set LOS Control to mask of RXLOSB_I */
1362                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc314, 0x0042);
1363                                 /* set LED1 to input mode (LED1 and LED2 share same LED) */
1364                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd006, 0x0007);
1365                                 /* set LED2 to RX link_status and activity */
1366                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd007, 0x000A);
1367                                 /* set LED3 to RX link_status */
1368                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd008, 0x0009);
1369
1370                                 /*
1371                                  * reset the res-calibration on t2
1372                                  * serdes; ensures it is stable after
1373                                  * the amcc phy is stable
1374                                  */
1375
1376                                 sds_common_control0  = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
1377                                 sds_common_control0 |= 0x1;
1378                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1379
1380                                 /* release the res-calibration reset */
1381                                 sds_common_control0 &= 0xfffffffe;
1382                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1383
1384                                 i = 0;
1385                                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1386                                                 && (i++ < 5000)) {
1387                                         /* mdelay(1); */
1388                                 }
1389
1390                                 /*
1391                                  * wait for link train done before moving on,
1392                                  * or will get an interupt storm
1393                                  */
1394                                 counter = 0;
1395                                 do {
1396                                         temp_phy_data = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
1397                                                                 (0x200 * (nesdev->mac_index & 1)));
1398                                         if (counter++ > 1000) {
1399                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from link train wait <this is bad, link didnt train!!!>\n");
1400                                                 break;
1401                                         }
1402                                         mdelay(1);
1403                                 } while (((temp_phy_data & 0x0f1f0000) != 0x0f0f0000));
1404                         }
1405                 }
1406         }
1407         return 0;
1408 }
1409
1410
1411 /**
1412  * nes_replenish_nic_rq
1413  */
1414 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1415 {
1416         unsigned long flags;
1417         dma_addr_t bus_address;
1418         struct sk_buff *skb;
1419         struct nes_hw_nic_rq_wqe *nic_rqe;
1420         struct nes_hw_nic *nesnic;
1421         struct nes_device *nesdev;
1422         u32 rx_wqes_posted = 0;
1423
1424         nesnic = &nesvnic->nic;
1425         nesdev = nesvnic->nesdev;
1426         spin_lock_irqsave(&nesnic->rq_lock, flags);
1427         if (nesnic->replenishing_rq !=0) {
1428                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1429                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1430                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1431                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1432                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1433                         add_timer(&nesvnic->rq_wqes_timer);
1434                 } else
1435                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1436                 return;
1437         }
1438         nesnic->replenishing_rq = 1;
1439         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1440         do {
1441                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1442                 if (skb) {
1443                         skb->dev = nesvnic->netdev;
1444
1445                         bus_address = pci_map_single(nesdev->pcidev,
1446                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1447
1448                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1449                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1450                                         cpu_to_le32(nesvnic->max_frame_size);
1451                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1452                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1453                                         cpu_to_le32((u32)bus_address);
1454                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1455                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1456                         nesnic->rx_skb[nesnic->rq_head] = skb;
1457                         nesnic->rq_head++;
1458                         nesnic->rq_head &= nesnic->rq_size - 1;
1459                         atomic_dec(&nesvnic->rx_skbs_needed);
1460                         barrier();
1461                         if (++rx_wqes_posted == 255) {
1462                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1463                                 rx_wqes_posted = 0;
1464                         }
1465                 } else {
1466                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1467                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1468                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1469                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1470                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1471                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1472                                 add_timer(&nesvnic->rq_wqes_timer);
1473                         } else
1474                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1475                         break;
1476                 }
1477         } while (atomic_read(&nesvnic->rx_skbs_needed));
1478         barrier();
1479         if (rx_wqes_posted)
1480                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1481         nesnic->replenishing_rq = 0;
1482 }
1483
1484
1485 /**
1486  * nes_rq_wqes_timeout
1487  */
1488 static void nes_rq_wqes_timeout(unsigned long parm)
1489 {
1490         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1491         printk("%s: Timer fired.\n", __func__);
1492         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1493         if (atomic_read(&nesvnic->rx_skbs_needed))
1494                 nes_replenish_nic_rq(nesvnic);
1495 }
1496
1497
1498 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1499                                void **tcph, u64 *hdr_flags, void *priv)
1500 {
1501         unsigned int ip_len;
1502         struct iphdr *iph;
1503         skb_reset_network_header(skb);
1504         iph = ip_hdr(skb);
1505         if (iph->protocol != IPPROTO_TCP)
1506                 return -1;
1507         ip_len = ip_hdrlen(skb);
1508         skb_set_transport_header(skb, ip_len);
1509         *tcph = tcp_hdr(skb);
1510
1511         *hdr_flags = LRO_IPV4 | LRO_TCP;
1512         *iphdr = iph;
1513         return 0;
1514 }
1515
1516
1517 /**
1518  * nes_init_nic_qp
1519  */
1520 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1521 {
1522         struct nes_hw_cqp_wqe *cqp_wqe;
1523         struct nes_hw_nic_sq_wqe *nic_sqe;
1524         struct nes_hw_nic_qp_context *nic_context;
1525         struct sk_buff *skb;
1526         struct nes_hw_nic_rq_wqe *nic_rqe;
1527         struct nes_vnic *nesvnic = netdev_priv(netdev);
1528         unsigned long flags;
1529         void *vmem;
1530         dma_addr_t pmem;
1531         u64 u64temp;
1532         int ret;
1533         u32 cqp_head;
1534         u32 counter;
1535         u32 wqe_count;
1536         u8 jumbomode=0;
1537
1538         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1539         nesvnic->nic_mem_size = 256 +
1540                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1541                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1542                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1543                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1544                         sizeof(struct nes_hw_nic_qp_context);
1545
1546         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1547                         &nesvnic->nic_pbase);
1548         if (!nesvnic->nic_vbase) {
1549                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1550                 return -ENOMEM;
1551         }
1552         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1553         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1554                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1555
1556         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1557                         ~(unsigned long)(256 - 1));
1558         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1559                         ~(unsigned long long)(256 - 1));
1560
1561         /* Setup the first Fragment buffers */
1562         nesvnic->nic.first_frag_vbase = vmem;
1563
1564         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1565                 nesvnic->nic.frag_paddr[counter] = pmem;
1566                 pmem += sizeof(struct nes_first_frag);
1567         }
1568
1569         /* setup the SQ */
1570         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1571
1572         nesvnic->nic.sq_vbase = (void *)vmem;
1573         nesvnic->nic.sq_pbase = pmem;
1574         nesvnic->nic.sq_head = 0;
1575         nesvnic->nic.sq_tail = 0;
1576         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1577         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1578                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1579                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1580                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1581                                 NES_NIC_SQ_WQE_COMPLETION);
1582                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1583                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1584                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1585                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1586                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1587                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1588         }
1589
1590         nesvnic->get_cqp_request = nes_get_cqp_request;
1591         nesvnic->post_cqp_request = nes_post_cqp_request;
1592         nesvnic->mcrq_mcast_filter = NULL;
1593
1594         spin_lock_init(&nesvnic->nic.sq_lock);
1595         spin_lock_init(&nesvnic->nic.rq_lock);
1596
1597         /* setup the RQ */
1598         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1599         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1600
1601
1602         nesvnic->nic.rq_vbase = vmem;
1603         nesvnic->nic.rq_pbase = pmem;
1604         nesvnic->nic.rq_head = 0;
1605         nesvnic->nic.rq_tail = 0;
1606         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1607
1608         /* setup the CQ */
1609         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1610         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1611
1612         if (nesdev->nesadapter->netdev_count > 2)
1613                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1614         else
1615                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1616
1617         nesvnic->nic_cq.cq_vbase = vmem;
1618         nesvnic->nic_cq.cq_pbase = pmem;
1619         nesvnic->nic_cq.cq_head = 0;
1620         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1621
1622         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1623
1624         /* Send CreateCQ request to CQP */
1625         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1626         cqp_head = nesdev->cqp.sq_head;
1627
1628         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1629         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1630
1631         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1632                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1633                         ((u32)nesvnic->nic_cq.cq_size << 16));
1634         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1635                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1636         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1637         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1638         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1639         u64temp = (unsigned long)&nesvnic->nic_cq;
1640         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1641         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1642                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1643         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1644         if (++cqp_head >= nesdev->cqp.sq_size)
1645                 cqp_head = 0;
1646         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1647         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1648
1649         /* Send CreateQP request to CQP */
1650         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1651         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1652                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1653                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1654         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1655                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1656                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1657         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1658                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1659         }
1660
1661         u64temp = (u64)nesvnic->nic.sq_pbase;
1662         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1663         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1664         u64temp = (u64)nesvnic->nic.rq_pbase;
1665         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1666         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1667
1668         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1669                         NES_CQP_QP_TYPE_NIC);
1670         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1671         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1672                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1673         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1674
1675         if (++cqp_head >= nesdev->cqp.sq_size)
1676                 cqp_head = 0;
1677         nesdev->cqp.sq_head = cqp_head;
1678
1679         barrier();
1680
1681         /* Ring doorbell (2 WQEs) */
1682         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1683
1684         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1685         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1686                         nesvnic->nic.qp_id);
1687
1688         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1689                         NES_EVENT_TIMEOUT);
1690         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1691                         nesvnic->nic.qp_id, ret);
1692         if (!ret) {
1693                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1694                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1695                                 nesvnic->nic_pbase);
1696                 return -EIO;
1697         }
1698
1699         /* Populate the RQ */
1700         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1701                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1702                 if (!skb) {
1703                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1704
1705                         nes_destroy_nic_qp(nesvnic);
1706                         return -ENOMEM;
1707                 }
1708
1709                 skb->dev = netdev;
1710
1711                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1712                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1713
1714                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1715                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1716                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1717                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1718                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1719                 nesvnic->nic.rx_skb[counter] = skb;
1720         }
1721
1722         wqe_count = NES_NIC_WQ_SIZE - 1;
1723         nesvnic->nic.rq_head = wqe_count;
1724         barrier();
1725         do {
1726                 counter = min(wqe_count, ((u32)255));
1727                 wqe_count -= counter;
1728                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1729         } while (wqe_count);
1730         init_timer(&nesvnic->rq_wqes_timer);
1731         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1732         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1733         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1734         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1735         {
1736                 nes_nic_init_timer(nesdev);
1737                 if (netdev->mtu > 1500)
1738                         jumbomode = 1;
1739                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1740         }
1741         nesvnic->lro_mgr.max_aggr       = NES_LRO_MAX_AGGR;
1742         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1743         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1744         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1745         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1746         nesvnic->lro_mgr.dev            = netdev;
1747         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1748         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1749         return 0;
1750 }
1751
1752
1753 /**
1754  * nes_destroy_nic_qp
1755  */
1756 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1757 {
1758         struct nes_device *nesdev = nesvnic->nesdev;
1759         struct nes_hw_cqp_wqe *cqp_wqe;
1760         struct nes_hw_nic_rq_wqe *nic_rqe;
1761         u64 wqe_frag;
1762         u32 cqp_head;
1763         unsigned long flags;
1764         int ret;
1765
1766         /* Free remaining NIC receive buffers */
1767         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1768                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1769                 wqe_frag  = (u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1770                 wqe_frag |= ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
1771                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1772                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1773                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1774                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1775         }
1776
1777         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1778
1779         /* Destroy NIC QP */
1780         cqp_head = nesdev->cqp.sq_head;
1781         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1782         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1783
1784         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1785                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1786         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1787                 nesvnic->nic.qp_id);
1788
1789         if (++cqp_head >= nesdev->cqp.sq_size)
1790                 cqp_head = 0;
1791
1792         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1793
1794         /* Destroy NIC CQ */
1795         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1796         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1797                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1798         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1799                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1800
1801         if (++cqp_head >= nesdev->cqp.sq_size)
1802                 cqp_head = 0;
1803
1804         nesdev->cqp.sq_head = cqp_head;
1805         barrier();
1806
1807         /* Ring doorbell (2 WQEs) */
1808         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1809
1810         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1811         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1812                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1813                         cqp_head, nesdev->cqp.sq_head,
1814                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1815
1816         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1817                         NES_EVENT_TIMEOUT);
1818
1819         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1820                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1821                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1822         if (!ret) {
1823                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1824                                 nesvnic->nic.qp_id);
1825         }
1826
1827         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1828                         nesvnic->nic_pbase);
1829 }
1830
1831 /**
1832  * nes_napi_isr
1833  */
1834 int nes_napi_isr(struct nes_device *nesdev)
1835 {
1836         struct nes_adapter *nesadapter = nesdev->nesadapter;
1837         u32 int_stat;
1838
1839         if (nesdev->napi_isr_ran) {
1840                 /* interrupt status has already been read in ISR */
1841                 int_stat = nesdev->int_stat;
1842         } else {
1843                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1844                 nesdev->int_stat = int_stat;
1845                 nesdev->napi_isr_ran = 1;
1846         }
1847
1848         int_stat &= nesdev->int_req;
1849         /* iff NIC, process here, else wait for DPC */
1850         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1851                 nesdev->napi_isr_ran = 0;
1852                 nes_write32(nesdev->regs + NES_INT_STAT,
1853                         (int_stat &
1854                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1855
1856                 /* Process the CEQs */
1857                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1858
1859                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1860                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1861                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1862                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1863                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1864                                 /* Enable Periodic timer interrupts */
1865                                 nesdev->int_req |= NES_INT_TIMER;
1866                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1867                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1868                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1869                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1870                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1871                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1872                         }
1873
1874                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1875                         {
1876                                 nes_nic_init_timer(nesdev);
1877                         }
1878                         /* Enable interrupts, except CEQs */
1879                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1880                 } else {
1881                         /* Enable interrupts, make sure timer is off */
1882                         nesdev->int_req &= ~NES_INT_TIMER;
1883                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1884                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1885                 }
1886                 nesdev->deepcq_count = 0;
1887                 return 1;
1888         } else {
1889                 return 0;
1890         }
1891 }
1892
1893
1894 /**
1895  * nes_dpc
1896  */
1897 void nes_dpc(unsigned long param)
1898 {
1899         struct nes_device *nesdev = (struct nes_device *)param;
1900         struct nes_adapter *nesadapter = nesdev->nesadapter;
1901         u32 counter;
1902         u32 loop_counter = 0;
1903         u32 int_status_bit;
1904         u32 int_stat;
1905         u32 timer_stat;
1906         u32 temp_int_stat;
1907         u32 intf_int_stat;
1908         u32 debug_error;
1909         u32 processed_intf_int = 0;
1910         u16 processed_timer_int = 0;
1911         u16 completion_ints = 0;
1912         u16 timer_ints = 0;
1913
1914         /* nes_debug(NES_DBG_ISR, "\n"); */
1915
1916         do {
1917                 timer_stat = 0;
1918                 if (nesdev->napi_isr_ran) {
1919                         nesdev->napi_isr_ran = 0;
1920                         int_stat = nesdev->int_stat;
1921                 } else
1922                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
1923                 if (processed_intf_int != 0)
1924                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
1925                 else
1926                         int_stat &= nesdev->int_req;
1927                 if (processed_timer_int == 0) {
1928                         processed_timer_int = 1;
1929                         if (int_stat & NES_INT_TIMER) {
1930                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
1931                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
1932                                         int_stat &= ~NES_INT_TIMER;
1933                                 }
1934                         }
1935                 } else {
1936                         int_stat &= ~NES_INT_TIMER;
1937                 }
1938
1939                 if (int_stat) {
1940                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
1941                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
1942                                 /* Ack the interrupts */
1943                                 nes_write32(nesdev->regs+NES_INT_STAT,
1944                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
1945                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1946                         }
1947
1948                         temp_int_stat = int_stat;
1949                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
1950                                 if (int_stat & int_status_bit) {
1951                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
1952                                         temp_int_stat &= ~int_status_bit;
1953                                         completion_ints = 1;
1954                                 }
1955                                 if (!(temp_int_stat & 0x0000ffff))
1956                                         break;
1957                                 int_status_bit <<= 1;
1958                         }
1959
1960                         /* Process the AEQ for this pci function */
1961                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
1962                         if (int_stat & int_status_bit) {
1963                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
1964                         }
1965
1966                         /* Process the MAC interrupt for this pci function */
1967                         int_status_bit = 1 << (24 + nesdev->mac_index);
1968                         if (int_stat & int_status_bit) {
1969                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
1970                         }
1971
1972                         if (int_stat & NES_INT_TIMER) {
1973                                 if (timer_stat & nesdev->timer_int_req) {
1974                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
1975                                                         (timer_stat & nesdev->timer_int_req) |
1976                                                         ~(nesdev->nesadapter->timer_int_req));
1977                                         timer_ints = 1;
1978                                 }
1979                         }
1980
1981                         if (int_stat & NES_INT_INTF) {
1982                                 processed_intf_int = 1;
1983                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
1984                                 intf_int_stat &= nesdev->intf_int_req;
1985                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
1986                                         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1987                                         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1988                                                         (u16)debug_error);
1989                                         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1990                                                         0x01010000 | (debug_error & 0x0000ffff));
1991                                         /* BUG(); */
1992                                         if (crit_err_count++ > 10)
1993                                                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
1994                                 }
1995                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
1996                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
1997                                         BUG();
1998                                 }
1999                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2000                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2001                                         BUG();
2002                                 }
2003                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2004                         }
2005
2006                         if (int_stat & NES_INT_TSW) {
2007                         }
2008                 }
2009                 /* Don't use the interface interrupt bit stay in loop */
2010                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2011                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2012         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2013
2014         if (timer_ints == 1) {
2015                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2016                         if (completion_ints == 0) {
2017                                 nesdev->timer_only_int_count++;
2018                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2019                                         nesdev->timer_only_int_count = 0;
2020                                         nesdev->int_req &= ~NES_INT_TIMER;
2021                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2022                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2023                                 } else {
2024                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2025                                 }
2026                         } else {
2027                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2028                                 {
2029                                         nes_nic_init_timer(nesdev);
2030                                 }
2031                                 nesdev->timer_only_int_count = 0;
2032                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2033                         }
2034                 } else {
2035                         nesdev->timer_only_int_count = 0;
2036                         nesdev->int_req &= ~NES_INT_TIMER;
2037                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2038                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2039                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2040                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2041                 }
2042         } else {
2043                 if ( (completion_ints == 1) &&
2044                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2045                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2046                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2047                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2048                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2049                         nesdev->timer_only_int_count = 0;
2050                         nesdev->int_req |= NES_INT_TIMER;
2051                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2052                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2053                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2054                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2055                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2056                 } else {
2057                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2058                 }
2059         }
2060         nesdev->deepcq_count = 0;
2061 }
2062
2063
2064 /**
2065  * nes_process_ceq
2066  */
2067 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2068 {
2069         u64 u64temp;
2070         struct nes_hw_cq *cq;
2071         u32 head;
2072         u32 ceq_size;
2073
2074         /* nes_debug(NES_DBG_CQ, "\n"); */
2075         head = ceq->ceq_head;
2076         ceq_size = ceq->ceq_size;
2077
2078         do {
2079                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2080                                 NES_CEQE_VALID) {
2081                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2082                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2083                         u64temp <<= 1;
2084                         cq = *((struct nes_hw_cq **)&u64temp);
2085                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2086                         barrier();
2087                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2088
2089                         /* call the event handler */
2090                         cq->ce_handler(nesdev, cq);
2091
2092                         if (++head >= ceq_size)
2093                                 head = 0;
2094                 } else {
2095                         break;
2096                 }
2097
2098         } while (1);
2099
2100         ceq->ceq_head = head;
2101 }
2102
2103
2104 /**
2105  * nes_process_aeq
2106  */
2107 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2108 {
2109         /* u64 u64temp; */
2110         u32 head;
2111         u32 aeq_size;
2112         u32 aeqe_misc;
2113         u32 aeqe_cq_id;
2114         struct nes_hw_aeqe volatile *aeqe;
2115
2116         head = aeq->aeq_head;
2117         aeq_size = aeq->aeq_size;
2118
2119         do {
2120                 aeqe = &aeq->aeq_vbase[head];
2121                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2122                         break;
2123                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2124                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2125                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2126                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2127                                 /* dealing with an accelerated QP related AE */
2128                                 /*
2129                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2130                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2131                                  */
2132                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2133                         } else {
2134                                 /* TODO: dealing with a CQP related AE */
2135                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2136                                                 (u16)(aeqe_misc >> 16));
2137                         }
2138                 }
2139
2140                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2141
2142                 if (++head >= aeq_size)
2143                         head = 0;
2144         }
2145         while (1);
2146         aeq->aeq_head = head;
2147 }
2148
2149 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2150 {
2151         struct nes_adapter *nesadapter = nesdev->nesadapter;
2152         u32 reset_value;
2153         u32 i=0;
2154         u32 u32temp;
2155
2156         if (nesadapter->hw_rev == NE020_REV) {
2157                 return;
2158         }
2159         mh_detected++;
2160
2161         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2162
2163         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2164                 reset_value |= 0x0000001d;
2165         else
2166                 reset_value |= 0x0000002d;
2167
2168         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2169                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2170                         nesadapter->link_interrupt_count[0] = 0;
2171                         nesadapter->link_interrupt_count[1] = 0;
2172                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2173                         if (0x00000040 & u32temp)
2174                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2175                         else
2176                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2177
2178                         reset_value |= 0x0000003d;
2179                 }
2180                 nesadapter->link_interrupt_count[mac_index] = 0;
2181         }
2182
2183         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2184
2185         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2186                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2187
2188         if (0x0000003d == (reset_value & 0x0000003d)) {
2189                 u32 pcs_control_status0, pcs_control_status1;
2190
2191                 for (i = 0; i < 10; i++) {
2192                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2193                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2194                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2195                              && (pcs_control_status0 & 0x00100000))
2196                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2197                                 && (pcs_control_status1 & 0x00100000)))
2198                                 continue;
2199                         else
2200                                 break;
2201                 }
2202                 if (10 == i) {
2203                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2204                         if (0x00000040 & u32temp)
2205                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2206                         else
2207                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2208
2209                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2210
2211                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2212                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2213                 }
2214         }
2215 }
2216
2217 /**
2218  * nes_process_mac_intr
2219  */
2220 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2221 {
2222         unsigned long flags;
2223         u32 pcs_control_status;
2224         struct nes_adapter *nesadapter = nesdev->nesadapter;
2225         struct nes_vnic *nesvnic;
2226         u32 mac_status;
2227         u32 mac_index = nesdev->mac_index;
2228         u32 u32temp;
2229         u16 phy_data;
2230         u16 temp_phy_data;
2231         u32 pcs_val  = 0x0f0f0000;
2232         u32 pcs_mask = 0x0f1f0000;
2233
2234         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2235         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2236                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2237                 return;
2238         }
2239         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2240         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2241
2242         /* ack the MAC interrupt */
2243         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2244         /* Clear the interrupt */
2245         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2246
2247         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2248
2249         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2250                 nesdev->link_status_interrupts++;
2251                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2252                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2253                         nes_reset_link(nesdev, mac_index);
2254                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2255                 }
2256                 /* read the PHY interrupt status register */
2257                 if (nesadapter->OneG_Mode) {
2258                         do {
2259                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2260                                                 nesadapter->phy_index[mac_index], &phy_data);
2261                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2262                                                 nesadapter->phy_index[mac_index], phy_data);
2263                         } while (phy_data&0x8000);
2264
2265                         temp_phy_data = 0;
2266                         do {
2267                                 nes_read_1G_phy_reg(nesdev, 0x11,
2268                                                 nesadapter->phy_index[mac_index], &phy_data);
2269                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2270                                                 nesadapter->phy_index[mac_index], phy_data);
2271                                 if (temp_phy_data == phy_data)
2272                                         break;
2273                                 temp_phy_data = phy_data;
2274                         } while (1);
2275
2276                         nes_read_1G_phy_reg(nesdev, 0x1e,
2277                                         nesadapter->phy_index[mac_index], &phy_data);
2278                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2279                                         nesadapter->phy_index[mac_index], phy_data);
2280
2281                         nes_read_1G_phy_reg(nesdev, 1,
2282                                         nesadapter->phy_index[mac_index], &phy_data);
2283                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2284                                         nesadapter->phy_index[mac_index], phy_data);
2285
2286                         if (temp_phy_data & 0x1000) {
2287                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2288                                 phy_data = 4;
2289                         } else {
2290                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2291                         }
2292                 }
2293                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2294                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2295                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2296
2297                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2298                         switch (mac_index) {
2299                         case 1:
2300                         case 3:
2301                                 pcs_control_status = nes_read_indexed(nesdev,
2302                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2303                                 break;
2304                         default:
2305                                 pcs_control_status = nes_read_indexed(nesdev,
2306                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2307                                 break;
2308                         }
2309                 } else {
2310                         pcs_control_status = nes_read_indexed(nesdev,
2311                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2312                         pcs_control_status = nes_read_indexed(nesdev,
2313                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2314                 }
2315
2316                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2317                                 mac_index, pcs_control_status);
2318                 if ((nesadapter->OneG_Mode) &&
2319                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2320                         u32temp = 0x01010000;
2321                         if (nesadapter->port_count > 2) {
2322                                 u32temp |= 0x02020000;
2323                         }
2324                         if ((pcs_control_status & u32temp)!= u32temp) {
2325                                 phy_data = 0;
2326                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2327                         }
2328                 } else {
2329                         switch (nesadapter->phy_type[mac_index]) {
2330                         case NES_PHY_TYPE_IRIS:
2331                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2332                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2333                                 u32temp = 20;
2334                                 do {
2335                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2336                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2337                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2338                                                 break;
2339                                         temp_phy_data = phy_data;
2340                                 } while (1);
2341                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2342                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2343                                 break;
2344
2345                         case NES_PHY_TYPE_ARGUS:
2346                                 /* clear the alarms */
2347                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2348                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2349                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2350                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2351                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2352                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2353                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2354                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2355                                 /* check link status */
2356                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2357                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2358                                 u32temp = 100;
2359                                 do {
2360                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2361
2362                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2363                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2364                                                 break;
2365                                         temp_phy_data = phy_data;
2366                                 } while (1);
2367                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2368                                         __func__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
2369                                 break;
2370
2371                         case NES_PHY_TYPE_PUMA_1G:
2372                                 if (mac_index < 2)
2373                                         pcs_val = pcs_mask = 0x01010000;
2374                                 else
2375                                         pcs_val = pcs_mask = 0x02020000;
2376                                 /* fall through */
2377                         default:
2378                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2379                                 break;
2380                         }
2381                 }
2382
2383                 if (phy_data & 0x0004) {
2384                         nesadapter->mac_link_down[mac_index] = 0;
2385                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2386                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2387                                                 nesvnic->linkup);
2388                                 if (nesvnic->linkup == 0) {
2389                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2390                                                         nesvnic->netdev->name, nesvnic->netdev);
2391                                         if (netif_queue_stopped(nesvnic->netdev))
2392                                                 netif_start_queue(nesvnic->netdev);
2393                                         nesvnic->linkup = 1;
2394                                         netif_carrier_on(nesvnic->netdev);
2395                                 }
2396                         }
2397                 } else {
2398                         nesadapter->mac_link_down[mac_index] = 1;
2399                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2400                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2401                                                 nesvnic->linkup);
2402                                 if (nesvnic->linkup == 1) {
2403                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2404                                                         nesvnic->netdev->name, nesvnic->netdev);
2405                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2406                                                 netif_stop_queue(nesvnic->netdev);
2407                                         nesvnic->linkup = 0;
2408                                         netif_carrier_off(nesvnic->netdev);
2409                                 }
2410                         }
2411                 }
2412         }
2413
2414         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2415 }
2416
2417
2418
2419 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2420 {
2421         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2422
2423         netif_rx_schedule(nesdev->netdev[nesvnic->netdev_index], &nesvnic->napi);
2424 }
2425
2426
2427 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2428 * getting out of nic_ce_handler
2429 */
2430 #define MAX_RQES_TO_PROCESS     384
2431
2432 /**
2433  * nes_nic_ce_handler
2434  */
2435 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2436 {
2437         u64 u64temp;
2438         dma_addr_t bus_address;
2439         struct nes_hw_nic *nesnic;
2440         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2441         struct nes_adapter *nesadapter = nesdev->nesadapter;
2442         struct nes_hw_nic_rq_wqe *nic_rqe;
2443         struct nes_hw_nic_sq_wqe *nic_sqe;
2444         struct sk_buff *skb;
2445         struct sk_buff *rx_skb;
2446         __le16 *wqe_fragment_length;
2447         u32 head;
2448         u32 cq_size;
2449         u32 rx_pkt_size;
2450         u32 cqe_count=0;
2451         u32 cqe_errv;
2452         u32 cqe_misc;
2453         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2454         u16 vlan_tag;
2455         u16 pkt_type;
2456         u16 rqes_processed = 0;
2457         u8 sq_cqes = 0;
2458         u8 nes_use_lro = 0;
2459
2460         head = cq->cq_head;
2461         cq_size = cq->cq_size;
2462         cq->cqes_pending = 1;
2463         if (nesvnic->netdev->features & NETIF_F_LRO)
2464                 nes_use_lro = 1;
2465         do {
2466                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2467                                 NES_NIC_CQE_VALID) {
2468                         nesnic = &nesvnic->nic;
2469                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2470                         if (cqe_misc & NES_NIC_CQE_SQ) {
2471                                 sq_cqes++;
2472                                 wqe_fragment_index = 1;
2473                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2474                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2475                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2476                                 /* bump past the vlan tag */
2477                                 wqe_fragment_length++;
2478                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2479                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2480                                                         wqe_fragment_index * 2]);
2481                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2482                                                         wqe_fragment_index * 2])) << 32;
2483                                         bus_address = (dma_addr_t)u64temp;
2484                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2485                                                 pci_unmap_single(nesdev->pcidev,
2486                                                                 bus_address,
2487                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2488                                                                 PCI_DMA_TODEVICE);
2489                                         }
2490                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2491                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2492                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2493                                                                                 wqe_fragment_index * 2]);
2494                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2495                                                                                 + wqe_fragment_index * 2])) <<32;
2496                                                         bus_address = (dma_addr_t)u64temp;
2497                                                         pci_unmap_page(nesdev->pcidev,
2498                                                                         bus_address,
2499                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2500                                                                         PCI_DMA_TODEVICE);
2501                                                 } else
2502                                                         break;
2503                                         }
2504                                         if (skb)
2505                                                 dev_kfree_skb_any(skb);
2506                                 }
2507                                 nesnic->sq_tail++;
2508                                 nesnic->sq_tail &= nesnic->sq_size-1;
2509                                 if (sq_cqes > 128) {
2510                                         barrier();
2511                                 /* restart the queue if it had been stopped */
2512                                 if (netif_queue_stopped(nesvnic->netdev))
2513                                         netif_wake_queue(nesvnic->netdev);
2514                                         sq_cqes = 0;
2515                                 }
2516                         } else {
2517                                 rqes_processed ++;
2518
2519                                 cq->rx_cqes_completed++;
2520                                 cq->rx_pkts_indicated++;
2521                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2522                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2523                                 /* Get the skb */
2524                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2525                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2526                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2527                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2528                                 pci_unmap_single(nesdev->pcidev, bus_address,
2529                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2530                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2531                                 /* rx_skb->len = rx_pkt_size; */
2532                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2533                                 skb_put(rx_skb, rx_pkt_size);
2534                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2535                                 nesnic->rq_tail++;
2536                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2537
2538                                 atomic_inc(&nesvnic->rx_skbs_needed);
2539                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2540                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2541                                                         cq->cq_number | (cqe_count << 16));
2542                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2543                                         nesdev->currcq_count += cqe_count;
2544                                         cqe_count = 0;
2545                                         nes_replenish_nic_rq(nesvnic);
2546                                 }
2547                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2548                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2549                                 rx_skb->ip_summed = CHECKSUM_NONE;
2550
2551                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2552                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2553                                         if ((cqe_errv &
2554                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2555                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2556                                                 if (nesvnic->rx_checksum_disabled == 0) {
2557                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2558                                                 }
2559                                         } else
2560                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2561                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2562                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2563
2564                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2565                                         if ((cqe_errv &
2566                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2567                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2568                                                 if (nesvnic->rx_checksum_disabled == 0) {
2569                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2570                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2571                                                                   nesvnic->netdev->name); */
2572                                                 }
2573                                         } else
2574                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2575                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2576                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2577                                         }
2578                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2579                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2580
2581                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2582                                         nes_cm_recv(rx_skb, nesvnic->netdev);
2583                                 } else {
2584                                         if ((cqe_misc & NES_NIC_CQE_TAG_VALID) && (nesvnic->vlan_grp != NULL)) {
2585                                                 vlan_tag = (u16)(le32_to_cpu(
2586                                                                 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2587                                                                 >> 16);
2588                                                 nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2589                                                                 nesvnic->netdev->name, vlan_tag);
2590                                                 if (nes_use_lro)
2591                                                         lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2592                                                                         nesvnic->vlan_grp, vlan_tag, NULL);
2593                                                 else
2594                                                         nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2595                                         } else {
2596                                                 if (nes_use_lro)
2597                                                         lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2598                                                 else
2599                                                         nes_netif_rx(rx_skb);
2600                                         }
2601                                 }
2602
2603                                 nesvnic->netdev->last_rx = jiffies;
2604                                 /* nesvnic->netstats.rx_packets++; */
2605                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2606                         }
2607
2608                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2609                         /* Accounting... */
2610                         cqe_count++;
2611                         if (++head >= cq_size)
2612                                 head = 0;
2613                         if (cqe_count == 255) {
2614                                 /* Replenish Nic CQ */
2615                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2616                                                 cq->cq_number | (cqe_count << 16));
2617                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2618                                 nesdev->currcq_count += cqe_count;
2619                                 cqe_count = 0;
2620                         }
2621
2622                         if (cq->rx_cqes_completed >= nesvnic->budget)
2623                                 break;
2624                 } else {
2625                         cq->cqes_pending = 0;
2626                         break;
2627                 }
2628
2629         } while (1);
2630
2631         if (nes_use_lro)
2632                 lro_flush_all(&nesvnic->lro_mgr);
2633         if (sq_cqes) {
2634                 barrier();
2635                 /* restart the queue if it had been stopped */
2636                 if (netif_queue_stopped(nesvnic->netdev))
2637                         netif_wake_queue(nesvnic->netdev);
2638         }
2639         cq->cq_head = head;
2640         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2641                         cq->cq_number, cqe_count, cq->cq_head); */
2642         cq->cqe_allocs_pending = cqe_count;
2643         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2644         {
2645                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2646                 nesdev->currcq_count += cqe_count;
2647                 nes_nic_tune_timer(nesdev);
2648         }
2649         if (atomic_read(&nesvnic->rx_skbs_needed))
2650                 nes_replenish_nic_rq(nesvnic);
2651 }
2652
2653
2654 /**
2655  * nes_cqp_ce_handler
2656  */
2657 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2658 {
2659         u64 u64temp;
2660         unsigned long flags;
2661         struct nes_hw_cqp *cqp = NULL;
2662         struct nes_cqp_request *cqp_request;
2663         struct nes_hw_cqp_wqe *cqp_wqe;
2664         u32 head;
2665         u32 cq_size;
2666         u32 cqe_count=0;
2667         u32 error_code;
2668         /* u32 counter; */
2669
2670         head = cq->cq_head;
2671         cq_size = cq->cq_size;
2672
2673         do {
2674                 /* process the CQE */
2675                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2676                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2677
2678                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2679                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2680                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2681                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2682                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2683                         cqp = *((struct nes_hw_cqp **)&u64temp);
2684
2685                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2686                         if (error_code) {
2687                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2688                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2689                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2690                                                 (u16)(error_code >> 16),
2691                                                 (u16)error_code);
2692                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2693                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2694                         }
2695
2696                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2697                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2698                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2699                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2700                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2701                         if (cqp_request) {
2702                                 if (cqp_request->waiting) {
2703                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2704                                         cqp_request->major_code = (u16)(error_code >> 16);
2705                                         cqp_request->minor_code = (u16)error_code;
2706                                         barrier();
2707                                         cqp_request->request_done = 1;
2708                                         wake_up(&cqp_request->waitq);
2709                                         if (atomic_dec_and_test(&cqp_request->refcount)) {
2710                                                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2711                                                                 cqp_request,
2712                                                                 le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2713                                                 if (cqp_request->dynamic) {
2714                                                         kfree(cqp_request);
2715                                                 } else {
2716                                                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2717                                                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2718                                                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2719                                                 }
2720                                         }
2721                                 } else if (cqp_request->callback) {
2722                                         /* Envoke the callback routine */
2723                                         cqp_request->cqp_callback(nesdev, cqp_request);
2724                                         if (cqp_request->dynamic) {
2725                                                 kfree(cqp_request);
2726                                         } else {
2727                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2728                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2729                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2730                                         }
2731                                 } else {
2732                                         nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2733                                                         cqp_request,
2734                                                         le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX]) & 0x3f);
2735                                         if (cqp_request->dynamic) {
2736                                                 kfree(cqp_request);
2737                                         } else {
2738                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2739                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2740                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2741                                         }
2742                                 }
2743                         } else {
2744                                 wake_up(&nesdev->cqp.waitq);
2745                         }
2746
2747                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2748                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2749                         if (++cqp->sq_tail >= cqp->sq_size)
2750                                 cqp->sq_tail = 0;
2751
2752                         /* Accounting... */
2753                         cqe_count++;
2754                         if (++head >= cq_size)
2755                                 head = 0;
2756                 } else {
2757                         break;
2758                 }
2759         } while (1);
2760         cq->cq_head = head;
2761
2762         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2763         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2764                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2765                         (nesdev->cqp.sq_size - 1)) != 1)) {
2766                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2767                                 struct nes_cqp_request, list);
2768                 list_del_init(&cqp_request->list);
2769                 head = nesdev->cqp.sq_head++;
2770                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2771                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2772                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2773                 barrier();
2774                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2775                         cpu_to_le32((u32)((unsigned long)cqp_request));
2776                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2777                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2778                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2779                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2780                 /* Ring doorbell (1 WQEs) */
2781                 barrier();
2782                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2783         }
2784         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2785
2786         /* Arm the CCQ */
2787         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2788                         cq->cq_number);
2789         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2790 }
2791
2792
2793 /**
2794  * nes_process_iwarp_aeqe
2795  */
2796 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2797                                    struct nes_hw_aeqe *aeqe)
2798 {
2799         u64 context;
2800         u64 aeqe_context = 0;
2801         unsigned long flags;
2802         struct nes_qp *nesqp;
2803         int resource_allocated;
2804         /* struct iw_cm_id *cm_id; */
2805         struct nes_adapter *nesadapter = nesdev->nesadapter;
2806         struct ib_event ibevent;
2807         /* struct iw_cm_event cm_event; */
2808         u32 aeq_info;
2809         u32 next_iwarp_state = 0;
2810         u16 async_event_id;
2811         u8 tcp_state;
2812         u8 iwarp_state;
2813
2814         nes_debug(NES_DBG_AEQ, "\n");
2815         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2816         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2817                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2818                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2819         } else {
2820                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2821                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2822                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2823                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
2824                 BUG_ON(!context);
2825         }
2826
2827         async_event_id = (u16)aeq_info;
2828         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2829         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2830         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2831                         " Tcp state = %s, iWARP state = %s\n",
2832                         async_event_id,
2833                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2834                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2835
2836         switch (async_event_id) {
2837                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2838                         nesqp = *((struct nes_qp **)&context);
2839                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2840                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2841                                 nes_add_ref(&nesqp->ibqp);
2842                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2843                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2844                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2845                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2846                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2847                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2848                                                 async_event_id, nesqp->last_aeq, tcp_state);
2849                         }
2850                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2851                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2852                                 /* FIN Received but tcp state or IB state moved on,
2853                                                 should expect a close complete */
2854                                 return;
2855                         }
2856                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2857                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2858                 case NES_AEQE_AEID_TERMINATE_SENT:
2859                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2860                 case NES_AEQE_AEID_RESET_SENT:
2861                         nesqp = *((struct nes_qp **)&context);
2862                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2863                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2864                         }
2865                         nes_add_ref(&nesqp->ibqp);
2866                         spin_lock_irqsave(&nesqp->lock, flags);
2867                         nesqp->hw_iwarp_state = iwarp_state;
2868                         nesqp->hw_tcp_state = tcp_state;
2869                         nesqp->last_aeq = async_event_id;
2870
2871                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2872                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2873                                 nesqp->hte_added = 0;
2874                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2875                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2876                                                 nesqp->hwqp.qp_id);
2877                                 nes_hw_modify_qp(nesdev, nesqp,
2878                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2879                                 spin_lock_irqsave(&nesqp->lock, flags);
2880                         }
2881
2882                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2883                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2884                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2885                                 switch (nesqp->hw_iwarp_state) {
2886                                         case NES_AEQE_IWARP_STATE_RTS:
2887                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2888                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2889                                                 break;
2890                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2891                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2892                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2893                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2894                                                         next_iwarp_state |= 0x02000000;
2895                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2896                                                 }
2897                                                 break;
2898                                         default:
2899                                                 next_iwarp_state = 0;
2900                                 }
2901                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2902                                 if (next_iwarp_state) {
2903                                         nes_add_ref(&nesqp->ibqp);
2904                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2905                                                         " also added another reference\n",
2906                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2907                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2908                                 }
2909                                 nes_cm_disconn(nesqp);
2910                         } else {
2911                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
2912                                         /* FIN Received but ib state not RTS,
2913                                                         close complete will be on its way */
2914                                         spin_unlock_irqrestore(&nesqp->lock, flags);
2915                                         nes_rem_ref(&nesqp->ibqp);
2916                                         return;
2917                                 }
2918                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2919                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2920                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
2921                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2922                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2923                                                         " also added another reference\n",
2924                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2925                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2926                                 }
2927                                 nes_cm_disconn(nesqp);
2928                         }
2929                         break;
2930                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
2931                         nesqp = *((struct nes_qp **)&context);
2932                         spin_lock_irqsave(&nesqp->lock, flags);
2933                         nesqp->hw_iwarp_state = iwarp_state;
2934                         nesqp->hw_tcp_state = tcp_state;
2935                         nesqp->last_aeq = async_event_id;
2936                         spin_unlock_irqrestore(&nesqp->lock, flags);
2937                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
2938                                         " event on QP%u \n  Q2 Data:\n",
2939                                         nesqp->hwqp.qp_id);
2940                         if (nesqp->ibqp.event_handler) {
2941                                 ibevent.device = nesqp->ibqp.device;
2942                                 ibevent.element.qp = &nesqp->ibqp;
2943                                 ibevent.event = IB_EVENT_QP_FATAL;
2944                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2945                         }
2946                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2947                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
2948                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2949                                 nes_add_ref(&nesqp->ibqp);
2950                                 nes_cm_disconn(nesqp);
2951                         } else {
2952                                 nesqp->in_disconnect = 0;
2953                                 wake_up(&nesqp->kick_waitq);
2954                         }
2955                         break;
2956                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
2957                         nesqp = *((struct nes_qp **)&context);
2958                         nes_add_ref(&nesqp->ibqp);
2959                         spin_lock_irqsave(&nesqp->lock, flags);
2960                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
2961                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2962                         nesqp->last_aeq = async_event_id;
2963                         if (nesqp->cm_id) {
2964                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2965                                                 " event on QP%u, remote IP = 0x%08X \n",
2966                                                 nesqp->hwqp.qp_id,
2967                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
2968                         } else {
2969                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2970                                                 " event on QP%u \n",
2971                                                 nesqp->hwqp.qp_id);
2972                         }
2973                         spin_unlock_irqrestore(&nesqp->lock, flags);
2974                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
2975                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2976                         if (nesqp->ibqp.event_handler) {
2977                                 ibevent.device = nesqp->ibqp.device;
2978                                 ibevent.element.qp = &nesqp->ibqp;
2979                                 ibevent.event = IB_EVENT_QP_FATAL;
2980                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2981                         }
2982                         break;
2983                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
2984                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
2985                                 nesqp = nesadapter->qp_table[le32_to_cpu(
2986                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2987                         } else {
2988                                 /* TODO: get the actual WQE and mask off wqe index */
2989                                 context &= ~((u64)511);
2990                                 nesqp = *((struct nes_qp **)&context);
2991                         }
2992                         spin_lock_irqsave(&nesqp->lock, flags);
2993                         nesqp->hw_iwarp_state = iwarp_state;
2994                         nesqp->hw_tcp_state = tcp_state;
2995                         nesqp->last_aeq = async_event_id;
2996                         spin_unlock_irqrestore(&nesqp->lock, flags);
2997                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
2998                                         nesqp->hwqp.qp_id);
2999                         if (nesqp->ibqp.event_handler) {
3000                                 ibevent.device = nesqp->ibqp.device;
3001                                 ibevent.element.qp = &nesqp->ibqp;
3002                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3003                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3004                         }
3005                         break;
3006                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3007                         nesqp = *((struct nes_qp **)&context);
3008                         spin_lock_irqsave(&nesqp->lock, flags);
3009                         nesqp->hw_iwarp_state = iwarp_state;
3010                         nesqp->hw_tcp_state = tcp_state;
3011                         nesqp->last_aeq = async_event_id;
3012                         spin_unlock_irqrestore(&nesqp->lock, flags);
3013                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
3014                                         nesqp->hwqp.qp_id);
3015                         if (nesqp->ibqp.event_handler) {
3016                                 ibevent.device = nesqp->ibqp.device;
3017                                 ibevent.element.qp = &nesqp->ibqp;
3018                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3019                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3020                         }
3021                         break;
3022                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3023                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
3024                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3025                         spin_lock_irqsave(&nesqp->lock, flags);
3026                         nesqp->hw_iwarp_state = iwarp_state;
3027                         nesqp->hw_tcp_state = tcp_state;
3028                         nesqp->last_aeq = async_event_id;
3029                         spin_unlock_irqrestore(&nesqp->lock, flags);
3030                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
3031                                         " nesqp = %p, AE reported %p\n",
3032                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
3033                         if (nesqp->ibqp.event_handler) {
3034                                 ibevent.device = nesqp->ibqp.device;
3035                                 ibevent.element.qp = &nesqp->ibqp;
3036                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3037                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3038                         }
3039                         break;
3040                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3041                         context <<= 1;
3042                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3043                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3044                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3045                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3046                         if (resource_allocated) {
3047                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3048                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3049                         }
3050                         break;
3051                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3052                         nesqp = nesadapter->qp_table[le32_to_cpu(
3053                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3054                         spin_lock_irqsave(&nesqp->lock, flags);
3055                         nesqp->hw_iwarp_state = iwarp_state;
3056                         nesqp->hw_tcp_state = tcp_state;
3057                         nesqp->last_aeq = async_event_id;
3058                         spin_unlock_irqrestore(&nesqp->lock, flags);
3059                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
3060                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
3061                                         nesqp->hwqp.qp_id);
3062                         if (nesqp->ibqp.event_handler) {
3063                                 ibevent.device = nesqp->ibqp.device;
3064                                 ibevent.element.qp = &nesqp->ibqp;
3065                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3066                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3067                         }
3068                         /* tell cm to disconnect, cm will queue work to thread */
3069                         nes_add_ref(&nesqp->ibqp);
3070                         nes_cm_disconn(nesqp);
3071                         break;
3072                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3073                         nesqp = *((struct nes_qp **)&context);
3074                         spin_lock_irqsave(&nesqp->lock, flags);
3075                         nesqp->hw_iwarp_state = iwarp_state;
3076                         nesqp->hw_tcp_state = tcp_state;
3077                         nesqp->last_aeq = async_event_id;
3078                         spin_unlock_irqrestore(&nesqp->lock, flags);
3079                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
3080                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
3081                                         nesqp->hwqp.qp_id);
3082                         if (nesqp->ibqp.event_handler) {
3083                                 ibevent.device = nesqp->ibqp.device;
3084                                 ibevent.element.qp = &nesqp->ibqp;
3085                                 ibevent.event = IB_EVENT_QP_FATAL;
3086                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3087                         }
3088                         /* tell cm to disconnect, cm will queue work to thread */
3089                         nes_add_ref(&nesqp->ibqp);
3090                         nes_cm_disconn(nesqp);
3091                         break;
3092                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3093                         nesqp = *((struct nes_qp **)&context);
3094                         spin_lock_irqsave(&nesqp->lock, flags);
3095                         nesqp->hw_iwarp_state = iwarp_state;
3096                         nesqp->hw_tcp_state = tcp_state;
3097                         nesqp->last_aeq = async_event_id;
3098                         spin_unlock_irqrestore(&nesqp->lock, flags);
3099                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
3100                                         " event on QP%u \n  Q2 Data:\n",
3101                                         nesqp->hwqp.qp_id);
3102                         if (nesqp->ibqp.event_handler) {
3103                                 ibevent.device = nesqp->ibqp.device;
3104                                 ibevent.element.qp = &nesqp->ibqp;
3105                                 ibevent.event = IB_EVENT_QP_FATAL;
3106                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3107                         }
3108                         /* tell cm to disconnect, cm will queue work to thread */
3109                         nes_add_ref(&nesqp->ibqp);
3110                         nes_cm_disconn(nesqp);
3111                         break;
3112                         /* TODO: additional AEs need to be here */
3113                 default:
3114                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3115                                         async_event_id);
3116                         break;
3117         }
3118
3119 }
3120
3121
3122 /**
3123  * nes_iwarp_ce_handler
3124  */
3125 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3126 {
3127         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3128
3129         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3130                         nescq->hw_cq.cq_number); */
3131         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3132
3133         if (nescq->ibcq.comp_handler)
3134                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3135
3136         return;
3137 }
3138
3139
3140 /**
3141  * nes_manage_apbvt()
3142  */
3143 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3144                 u32 nic_index, u32 add_port)
3145 {
3146         struct nes_device *nesdev = nesvnic->nesdev;
3147         struct nes_hw_cqp_wqe *cqp_wqe;
3148         unsigned long flags;
3149         struct nes_cqp_request *cqp_request;
3150         int ret = 0;
3151         u16 major_code;
3152
3153         /* Send manage APBVT request to CQP */
3154         cqp_request = nes_get_cqp_request(nesdev);
3155         if (cqp_request == NULL) {
3156                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3157                 return -ENOMEM;
3158         }
3159         cqp_request->waiting = 1;
3160         cqp_wqe = &cqp_request->cqp_wqe;
3161
3162         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3163                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3164                         accel_local_port, accel_local_port, nic_index);
3165
3166         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3167         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3168                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3169         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3170                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3171
3172         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3173
3174         atomic_set(&cqp_request->refcount, 2);
3175         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3176
3177         if (add_port == NES_MANAGE_APBVT_ADD)
3178                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3179                                 NES_EVENT_TIMEOUT);
3180         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3181                         ret, cqp_request->major_code, cqp_request->minor_code);
3182         major_code = cqp_request->major_code;
3183         if (atomic_dec_and_test(&cqp_request->refcount)) {
3184                 if (cqp_request->dynamic) {
3185                         kfree(cqp_request);
3186                 } else {
3187                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
3188                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3189                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3190                 }
3191         }
3192         if (!ret)
3193                 return -ETIME;
3194         else if (major_code)
3195                 return -EIO;
3196         else
3197                 return 0;
3198 }
3199
3200
3201 /**
3202  * nes_manage_arp_cache
3203  */
3204 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3205                 u32 ip_addr, u32 action)
3206 {
3207         struct nes_hw_cqp_wqe *cqp_wqe;
3208         struct nes_vnic *nesvnic = netdev_priv(netdev);
3209         struct nes_device *nesdev;
3210         struct nes_cqp_request *cqp_request;
3211         int arp_index;
3212
3213         nesdev = nesvnic->nesdev;
3214         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3215         if (arp_index == -1) {
3216                 return;
3217         }
3218
3219         /* update the ARP entry */
3220         cqp_request = nes_get_cqp_request(nesdev);
3221         if (cqp_request == NULL) {
3222                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3223                 return;
3224         }
3225         cqp_request->waiting = 0;
3226         cqp_wqe = &cqp_request->cqp_wqe;
3227         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3228
3229         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3230                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3231         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3232                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3233         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3234
3235         if (action == NES_ARP_ADD) {
3236                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3237                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3238                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3239                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3240                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3241                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3242         } else {
3243                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3244                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3245         }
3246
3247         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3248                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3249
3250         atomic_set(&cqp_request->refcount, 1);
3251         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3252 }
3253
3254
3255 /**
3256  * flush_wqes
3257  */
3258 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3259                 u32 which_wq, u32 wait_completion)
3260 {
3261         unsigned long flags;
3262         struct nes_cqp_request *cqp_request;
3263         struct nes_hw_cqp_wqe *cqp_wqe;
3264         int ret;
3265
3266         cqp_request = nes_get_cqp_request(nesdev);
3267         if (cqp_request == NULL) {
3268                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3269                 return;
3270         }
3271         if (wait_completion) {
3272                 cqp_request->waiting = 1;
3273                 atomic_set(&cqp_request->refcount, 2);
3274         } else {
3275                 cqp_request->waiting = 0;
3276         }
3277         cqp_wqe = &cqp_request->cqp_wqe;
3278         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3279
3280         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3281                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3282         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3283
3284         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3285
3286         if (wait_completion) {
3287                 /* Wait for CQP */
3288                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3289                                 NES_EVENT_TIMEOUT);
3290                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3291                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3292                                 ret, cqp_request->major_code, cqp_request->minor_code);
3293                 if (atomic_dec_and_test(&cqp_request->refcount)) {
3294                         if (cqp_request->dynamic) {
3295                                 kfree(cqp_request);
3296                         } else {
3297                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
3298                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3299                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3300                         }
3301                 }
3302         }
3303 }