drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Intel-NE, Inc.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static int wide_ppm_offset;
  50 module_param(wide_ppm_offset, int, 0644);
  51 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  52
  53 static u32 crit_err_count;
  54 u32 int_mod_timer_init;
  55 u32 int_mod_cq_depth_256;
  56 u32 int_mod_cq_depth_128;
  57 u32 int_mod_cq_depth_32;
  58 u32 int_mod_cq_depth_24;
  59 u32 int_mod_cq_depth_16;
  60 u32 int_mod_cq_depth_4;
  61 u32 int_mod_cq_depth_1;
  62 static const u8 nes_max_critical_error_count = 100;
  63 #include "nes_cm.h"
  64
  65 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  66 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  67 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  68                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  69 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  70 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  71 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  72 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  73                                    struct nes_hw_aeqe *aeqe);
  74 static void process_critical_error(struct nes_device *nesdev);
  75 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  76 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  77
  78 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  79 static unsigned char *nes_iwarp_state_str[] = {
  80         "Non-Existant",
  81         "Idle",
  82         "RTS",
  83         "Closing",
  84         "RSVD1",
  85         "Terminate",
  86         "Error",
  87         "RSVD2",
  88 };
  89
  90 static unsigned char *nes_tcp_state_str[] = {
  91         "Non-Existant",
  92         "Closed",
  93         "Listen",
  94         "SYN Sent",
  95         "SYN Rcvd",
  96         "Established",
  97         "Close Wait",
  98         "FIN Wait 1",
  99         "Closing",
 100         "Last Ack",
 101         "FIN Wait 2",
 102         "Time Wait",
 103         "RSVD1",
 104         "RSVD2",
 105         "RSVD3",
 106         "RSVD4",
 107 };
 108 #endif
 109
 110
 111 /**
 112  * nes_nic_init_timer_defaults
 113  */
 114 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 115 {
 116         unsigned long flags;
 117         struct nes_adapter *nesadapter = nesdev->nesadapter;
 118         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 119
 120         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 121
 122         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 123         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 124         if (jumbomode) {
 125                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 126                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 127                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 128         } else {
 129                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 130                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 131                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 132         }
 133
 134         /* todo use netdev->mtu to set thresholds */
 135         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 136 }
 137
 138
 139 /**
 140  * nes_nic_init_timer
 141  */
 142 static void  nes_nic_init_timer(struct nes_device *nesdev)
 143 {
 144         unsigned long flags;
 145         struct nes_adapter *nesadapter = nesdev->nesadapter;
 146         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 147
 148         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 149
 150         if (shared_timer->timer_in_use_old == 0) {
 151                 nesdev->deepcq_count = 0;
 152                 shared_timer->timer_direction_upward = 0;
 153                 shared_timer->timer_direction_downward = 0;
 154                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 155                 shared_timer->timer_in_use_old = 0;
 156
 157         }
 158         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 159                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 160                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 161                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 162         }
 163         /* todo use netdev->mtu to set thresholds */
 164         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 165 }
 166
 167
 168 /**
 169  * nes_nic_tune_timer
 170  */
 171 static void nes_nic_tune_timer(struct nes_device *nesdev)
 172 {
 173         unsigned long flags;
 174         struct nes_adapter *nesadapter = nesdev->nesadapter;
 175         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 176         u16 cq_count = nesdev->currcq_count;
 177
 178         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 179
 180         if (shared_timer->cq_count_old <= cq_count)
 181                 shared_timer->cq_direction_downward = 0;
 182         else
 183                 shared_timer->cq_direction_downward++;
 184         shared_timer->cq_count_old = cq_count;
 185         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 186                 if (cq_count <= shared_timer->threshold_low &&
 187                     shared_timer->threshold_low > 4) {
 188                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 189                         shared_timer->cq_direction_downward=0;
 190                         nesdev->currcq_count = 0;
 191                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 192                         return;
 193                 }
 194         }
 195
 196         if (cq_count > 1) {
 197                 nesdev->deepcq_count += cq_count;
 198                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 199                         shared_timer->timer_direction_upward++;
 200                         shared_timer->timer_direction_downward = 0;
 201                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 202                         shared_timer->timer_direction_upward = 0;
 203                         shared_timer->timer_direction_downward = 0;
 204                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 205                         shared_timer->timer_direction_downward++;
 206                         shared_timer->timer_direction_upward = 0;
 207                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 208                         shared_timer->timer_in_use -= 2;
 209                         shared_timer->timer_direction_upward = 0;
 210                         shared_timer->timer_direction_downward++;
 211                 } else {
 212                         shared_timer->timer_in_use -= 4;
 213                         shared_timer->timer_direction_upward = 0;
 214                         shared_timer->timer_direction_downward++;
 215                 }
 216
 217                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 218                         shared_timer->timer_in_use += 3;
 219                         shared_timer->timer_direction_upward = 0;
 220                         shared_timer->timer_direction_downward = 0;
 221                 }
 222                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 223                         shared_timer->timer_in_use -= 4 ;
 224                         shared_timer->timer_direction_downward = 0;
 225                         shared_timer->timer_direction_upward = 0;
 226                 }
 227         }
 228
 229         /* boundary checking */
 230         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 231                 shared_timer->timer_in_use = shared_timer->threshold_high;
 232         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 233                 shared_timer->timer_in_use = shared_timer->threshold_low;
 234
 235         nesdev->currcq_count = 0;
 236
 237         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 238 }
 239
 240
 241 /**
 242  * nes_init_adapter - initialize adapter
 243  */
 244 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 245         struct nes_adapter *nesadapter = NULL;
 246         unsigned long num_pds;
 247         u32 u32temp;
 248         u32 port_count;
 249         u16 max_rq_wrs;
 250         u16 max_sq_wrs;
 251         u32 max_mr;
 252         u32 max_256pbl;
 253         u32 max_4kpbl;
 254         u32 max_qp;
 255         u32 max_irrq;
 256         u32 max_cq;
 257         u32 hte_index_mask;
 258         u32 adapter_size;
 259         u32 arp_table_size;
 260         u16 vendor_id;
 261         u16 device_id;
 262         u8  OneG_Mode;
 263         u8  func_index;
 264
 265         /* search the list of existing adapters */
 266         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 267                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 268                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 269                                 nesdev->pcidev->devfn,
 270                                 PCI_SLOT(nesadapter->devfn),
 271                                 nesadapter->bus_number,
 272                                 PCI_SLOT(nesdev->pcidev->devfn),
 273                                 nesdev->pcidev->bus->number );
 274                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 275                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 276                         nesadapter->ref_count++;
 277                         return nesadapter;
 278                 }
 279         }
 280
 281         /* no adapter found */
 282         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 283         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 284                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 285                                 hw_rev);
 286                 return NULL;
 287         }
 288
 289         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 290                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 291                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 292                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 293                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 294
 295         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 296
 297
 298         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 299                 return NULL;
 300
 301         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 302         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 303
 304         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 305         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 306                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 307                                 max_qp, u32temp);
 308                 max_qp = (u32)1 << (u32temp & 0x001f);
 309         }
 310
 311         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 312         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 313                         max_qp, hte_index_mask);
 314
 315         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 316
 317         max_irrq = 1 << (u32temp & 0x001f);
 318
 319         if (max_qp > max_irrq) {
 320                 max_qp = max_irrq;
 321                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 322                                 max_qp);
 323         }
 324
 325         /* there should be no reason to allocate more pds than qps */
 326         if (num_pds > max_qp)
 327                 num_pds = max_qp;
 328
 329         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 330         max_mr = (u32)8192 << (u32temp & 0x7);
 331
 332         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 333         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 334         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 335         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 336
 337         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 338         arp_table_size = 1 << u32temp;
 339
 340         adapter_size = (sizeof(struct nes_adapter) +
 341                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 342         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 343         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 344         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 347         adapter_size += sizeof(struct nes_qp **) * max_qp;
 348
 349         /* allocate a new adapter struct */
 350         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 351         if (nesadapter == NULL) {
 352                 return NULL;
 353         }
 354
 355         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 356                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 357
 358         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 359                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 360                 kfree(nesadapter);
 361                 return NULL;
 362         }
 363
 364         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 365                                 (nesadapter->mac_addr_low >> 24);
 366
 367         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 368                                  PCI_DEVICE_ID, &device_id);
 369         nesadapter->vendor_part_id = device_id;
 370
 371         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 372                                                         OneG_Mode)) {
 373                 kfree(nesadapter);
 374                 return NULL;
 375         }
 376         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 377
 378         memset(nesadapter->pft_mcast_map, 255,
 379                sizeof nesadapter->pft_mcast_map);
 380
 381         /* populate the new nesadapter */
 382         nesadapter->devfn = nesdev->pcidev->devfn;
 383         nesadapter->bus_number = nesdev->pcidev->bus->number;
 384         nesadapter->ref_count = 1;
 385         nesadapter->timer_int_req = 0xffff0000;
 386         nesadapter->OneG_Mode = OneG_Mode;
 387         nesadapter->doorbell_start = nesdev->doorbell_region;
 388
 389         /* nesadapter->tick_delta = clk_divisor; */
 390         nesadapter->hw_rev = hw_rev;
 391         nesadapter->port_count = port_count;
 392
 393         nesadapter->max_qp = max_qp;
 394         nesadapter->hte_index_mask = hte_index_mask;
 395         nesadapter->max_irrq = max_irrq;
 396         nesadapter->max_mr = max_mr;
 397         nesadapter->max_256pbl = max_256pbl - 1;
 398         nesadapter->max_4kpbl = max_4kpbl - 1;
 399         nesadapter->max_cq = max_cq;
 400         nesadapter->free_256pbl = max_256pbl - 1;
 401         nesadapter->free_4kpbl = max_4kpbl - 1;
 402         nesadapter->max_pd = num_pds;
 403         nesadapter->arp_table_size = arp_table_size;
 404
 405         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 406         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 407                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 408                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 409                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 410         } else {
 411                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 412                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 413                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 414                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 415         }
 416         /* Setup and enable the periodic timer */
 417         if (nesadapter->et_rx_coalesce_usecs_irq)
 418                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 419                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 420         else
 421                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 422
 423         nesadapter->base_pd = 1;
 424
 425         nesadapter->device_cap_flags =
 426                 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
 427
 428         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 429                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 430         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 431         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 432         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 433         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 434         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 435
 436
 437         /* mark the usual suspect QPs and CQs as in use */
 438         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 439                 set_bit(u32temp, nesadapter->allocated_qps);
 440                 set_bit(u32temp, nesadapter->allocated_cqs);
 441         }
 442
 443         for (u32temp = 0; u32temp < 20; u32temp++)
 444                 set_bit(u32temp, nesadapter->allocated_pds);
 445         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 446
 447         max_rq_wrs = ((u32temp >> 8) & 3);
 448         switch (max_rq_wrs) {
 449                 case 0:
 450                         max_rq_wrs = 4;
 451                         break;
 452                 case 1:
 453                         max_rq_wrs = 16;
 454                         break;
 455                 case 2:
 456                         max_rq_wrs = 32;
 457                         break;
 458                 case 3:
 459                         max_rq_wrs = 512;
 460                         break;
 461         }
 462
 463         max_sq_wrs = (u32temp & 3);
 464         switch (max_sq_wrs) {
 465                 case 0:
 466                         max_sq_wrs = 4;
 467                         break;
 468                 case 1:
 469                         max_sq_wrs = 16;
 470                         break;
 471                 case 2:
 472                         max_sq_wrs = 32;
 473                         break;
 474                 case 3:
 475                         max_sq_wrs = 512;
 476                         break;
 477         }
 478         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 479         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 480
 481         nesadapter->max_sge = 4;
 482         nesadapter->max_cqe = 32767;
 483
 484         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 485                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 486                 kfree(nesadapter);
 487                 return NULL;
 488         }
 489
 490         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 491         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 492                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 493
 494         /* setup port configuration */
 495         if (nesadapter->port_count == 1) {
 496                 nesadapter->log_port = 0x00000000;
 497                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 498                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 499                 else
 500                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 501         } else {
 502                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 503                         nesadapter->log_port = 0x000000D8;
 504                 } else {
 505                         if (nesadapter->port_count == 2)
 506                                 nesadapter->log_port = 0x00000044;
 507                         else
 508                                 nesadapter->log_port = 0x000000e4;
 509                 }
 510                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 511         }
 512
 513         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 514                                                 nesadapter->log_port);
 515         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 516                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 517
 518         spin_lock_init(&nesadapter->resource_lock);
 519         spin_lock_init(&nesadapter->phy_lock);
 520         spin_lock_init(&nesadapter->pbl_lock);
 521         spin_lock_init(&nesadapter->periodic_timer_lock);
 522
 523         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 524         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 525         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 526         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 527
 528         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 529                 u32 pcs_control_status0, pcs_control_status1;
 530                 u32 reset_value;
 531                 u32 i = 0;
 532                 u32 int_cnt = 0;
 533                 u32 ext_cnt = 0;
 534                 unsigned long flags;
 535                 u32 j = 0;
 536
 537                 pcs_control_status0 = nes_read_indexed(nesdev,
 538                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 539                 pcs_control_status1 = nes_read_indexed(nesdev,
 540                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 541
 542                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 543                         pcs_control_status0 = nes_read_indexed(nesdev,
 544                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 545                         pcs_control_status1 = nes_read_indexed(nesdev,
 546                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 547                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 548                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 549                                 int_cnt++;
 550                         msleep(1);
 551                 }
 552                 if (int_cnt > 1) {
 553                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 554                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 555                         mh_detected++;
 556                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 557                         reset_value |= 0x0000003d;
 558                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 559
 560                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 561                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 562                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 563
 564                         pcs_control_status0 = nes_read_indexed(nesdev,
 565                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 566                         pcs_control_status1 = nes_read_indexed(nesdev,
 567                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 568
 569                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 570                                 pcs_control_status0 = nes_read_indexed(nesdev,
 571                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 572                                 pcs_control_status1 = nes_read_indexed(nesdev,
 573                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 574                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 575                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 576                                         if (++ext_cnt > int_cnt) {
 577                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 578                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 579                                                                 0x0000F088);
 580                                                 mh_detected++;
 581                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 582                                                 reset_value |= 0x0000003d;
 583                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 584
 585                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 586                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 587                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 588                                                 break;
 589                                         }
 590                                 }
 591                                 msleep(1);
 592                         }
 593                 }
 594         }
 595
 596         if (nesadapter->hw_rev == NE020_REV) {
 597                 init_timer(&nesadapter->mh_timer);
 598                 nesadapter->mh_timer.function = nes_mh_fix;
 599                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 600                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 601                 add_timer(&nesadapter->mh_timer);
 602         } else {
 603                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 604         }
 605
 606         init_timer(&nesadapter->lc_timer);
 607         nesadapter->lc_timer.function = nes_clc;
 608         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 609         nesadapter->lc_timer.data = (unsigned long)nesdev;
 610         add_timer(&nesadapter->lc_timer);
 611
 612         list_add_tail(&nesadapter->list, &nes_adapter_list);
 613
 614         for (func_index = 0; func_index < 8; func_index++) {
 615                 pci_bus_read_config_word(nesdev->pcidev->bus,
 616                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 617                                         func_index), 0, &vendor_id);
 618                 if (vendor_id == 0xffff)
 619                         break;
 620         }
 621         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 622                 func_index, pci_name(nesdev->pcidev));
 623         nesadapter->adapter_fcn_count = func_index;
 624
 625         return nesadapter;
 626 }
 627
 628
 629 /**
 630  * nes_reset_adapter_ne020
 631  */
 632 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 633 {
 634         u32 port_count;
 635         u32 u32temp;
 636         u32 i;
 637
 638         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 639         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 640         /* TODO: assuming that both SERDES are set the same for now */
 641         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 642         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 643                         u32temp, port_count);
 644         if (*OneG_Mode)
 645                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 646         u32temp &= 0xff00ffc0;
 647         switch (port_count) {
 648                 case 1:
 649                         u32temp |= 0x00ee0000;
 650                         break;
 651                 case 2:
 652                         u32temp |= 0x00cc0000;
 653                         break;
 654                 case 4:
 655                         u32temp |= 0x00000000;
 656                         break;
 657                 default:
 658                         return 0;
 659                         break;
 660         }
 661
 662         /* check and do full reset if needed */
 663         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 664                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 665                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 666
 667                 i = 0;
 668                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 669                         mdelay(1);
 670                 if (i >= 10000) {
 671                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 672                         return 0;
 673                 }
 674
 675                 i = 0;
 676                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 677                         mdelay(1);
 678                 if (i >= 10000) {
 679                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 680                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 681                         return 0;
 682                 }
 683         }
 684
 685         /* port reset */
 686         switch (port_count) {
 687                 case 1:
 688                         u32temp |= 0x00ee0010;
 689                         break;
 690                 case 2:
 691                         u32temp |= 0x00cc0030;
 692                         break;
 693                 case 4:
 694                         u32temp |= 0x00000030;
 695                         break;
 696         }
 697
 698         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 699         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 700
 701         i = 0;
 702         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 703                 mdelay(1);
 704         if (i >= 10000) {
 705                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 706                 return 0;
 707         }
 708
 709         /* serdes 0 */
 710         i = 0;
 711         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 712                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 713                 mdelay(1);
 714         if (i >= 5000) {
 715                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 716                 return 0;
 717         }
 718
 719         /* serdes 1 */
 720         if (port_count > 1) {
 721                 i = 0;
 722                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 723                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 724                         mdelay(1);
 725                 if (i >= 5000) {
 726                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 727                         return 0;
 728                 }
 729         }
 730
 731         return port_count;
 732 }
 733
 734
 735 /**
 736  * nes_init_serdes
 737  */
 738 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 739                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 740 {
 741         int i;
 742         u32 u32temp;
 743         u32 sds;
 744
 745         if (hw_rev != NE020_REV) {
 746                 /* init serdes 0 */
 747                 if (wide_ppm_offset && (nesadapter->phy_type[0] == NES_PHY_TYPE_CX4))
 748                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 749                 else
 750                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 751
 752                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 753                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 754                         sds |= 0x00000100;
 755                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 756                 }
 757                 if (!OneG_Mode)
 758                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 759
 760                 if (port_count < 2)
 761                         return 0;
 762
 763                 /* init serdes 1 */
 764                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 765                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 766
 767                 switch (nesadapter->phy_type[1]) {
 768                 case NES_PHY_TYPE_ARGUS:
 769                 case NES_PHY_TYPE_SFP_D:
 770                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 771                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 772                         break;
 773                 case NES_PHY_TYPE_CX4:
 774                         if (wide_ppm_offset)
 775                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 776                         break;
 777                 case NES_PHY_TYPE_PUMA_1G:
 778                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 779                         sds |= 0x000000100;
 780                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 781                 }
 782                 if (!OneG_Mode) {
 783                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 784                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 785                         sds &= 0xFFFFFFBF;
 786                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 787                 }
 788         } else {
 789                 /* init serdes 0 */
 790                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 791                 i = 0;
 792                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 793                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 794                         mdelay(1);
 795                 if (i >= 5000) {
 796                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 797                         return 1;
 798                 }
 799                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 800                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 801                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 802                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 803                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 804                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 805                 if (OneG_Mode)
 806                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 807                 else
 808                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 809
 810                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 811                 if (port_count > 1) {
 812                         /* init serdes 1 */
 813                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 814                         i = 0;
 815                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 816                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 817                                 mdelay(1);
 818                         if (i >= 5000) {
 819                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 820                                 /* return 1; */
 821                         }
 822                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 823                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 824                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 825                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 826                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 827                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 828                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 829                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 830                 }
 831         }
 832         return 0;
 833 }
 834
 835
 836 /**
 837  * nes_init_csr_ne020
 838  * Initialize registers for ne020 hardware
 839  */
 840 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 841 {
 842         u32 u32temp;
 843
 844         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 845
 846         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 847         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 848         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 849         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 850         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 851         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 852         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 853         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 854
 855         /* TODO: move these MAC register settings to NIC bringup */
 856         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 857         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 858         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 859         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 860         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 861         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 862         if (port_count > 1) {
 863                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 864                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 865                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 866                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 867                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 868                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 869                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 870         }
 871         if (port_count > 2) {
 872                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 873                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 874                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 875                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 876                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 877                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 878                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 879
 880                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 881                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 882                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 883                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 884                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 885                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 886                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 887         }
 888
 889         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 890         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 891         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 892                                                          0x00000001);
 893         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 894         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 895         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 896         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 897         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 898
 899         /* TODO: move this to code, get from EEPROM */
 900         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 901         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 902         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 903
 904         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 905         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 906
 907         if (hw_rev != NE020_REV) {
 908                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 909                 u32temp |= 0x80000000;
 910                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 911                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 912                 u32temp &= 0x7fffffff;
 913                 u32temp |= 0x7fff0010;
 914                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 915                 if (port_count > 1) {
 916                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 917                         u32temp &= 0x7fffffff;
 918                         u32temp |= 0x7fff0010;
 919                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 920                 }
 921         }
 922 }
 923
 924
 925 /**
 926  * nes_destroy_adapter - destroy the adapter structure
 927  */
 928 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 929 {
 930         struct nes_adapter *tmp_adapter;
 931
 932         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 933                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 934                                 tmp_adapter);
 935         }
 936
 937         nesadapter->ref_count--;
 938         if (!nesadapter->ref_count) {
 939                 if (nesadapter->hw_rev == NE020_REV) {
 940                         del_timer(&nesadapter->mh_timer);
 941                 }
 942                 del_timer(&nesadapter->lc_timer);
 943
 944                 list_del(&nesadapter->list);
 945                 kfree(nesadapter);
 946         }
 947 }
 948
 949
 950 /**
 951  * nes_init_cqp
 952  */
 953 int nes_init_cqp(struct nes_device *nesdev)
 954 {
 955         struct nes_adapter *nesadapter = nesdev->nesadapter;
 956         struct nes_hw_cqp_qp_context *cqp_qp_context;
 957         struct nes_hw_cqp_wqe *cqp_wqe;
 958         struct nes_hw_ceq *ceq;
 959         struct nes_hw_ceq *nic_ceq;
 960         struct nes_hw_aeq *aeq;
 961         void *vmem;
 962         dma_addr_t pmem;
 963         u32 count=0;
 964         u32 cqp_head;
 965         u64 u64temp;
 966         u32 u32temp;
 967
 968         /* allocate CQP memory */
 969         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 970         /* SQ is 512 byte aligned, others are 256 byte aligned */
 971         nesdev->cqp_mem_size = 512 +
 972                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 973                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 974                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 975                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 976                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 977                         sizeof(struct nes_hw_cqp_qp_context);
 978
 979         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 980                         &nesdev->cqp_pbase);
 981         if (!nesdev->cqp_vbase) {
 982                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 983                 return -ENOMEM;
 984         }
 985         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 986
 987         /* Allocate a twice the number of CQP requests as the SQ size */
 988         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 989                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 990         if (nesdev->nes_cqp_requests == NULL) {
 991                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 992                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 993                                 nesdev->cqp.sq_pbase);
 994                 return -ENOMEM;
 995         }
 996
 997         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 998                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 999
1000         spin_lock_init(&nesdev->cqp.lock);
1001         init_waitqueue_head(&nesdev->cqp.waitq);
1002
1003         /* Setup Various Structures */
1004         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1005                         ~(unsigned long)(512 - 1));
1006         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1007                         ~(unsigned long long)(512 - 1));
1008
1009         nesdev->cqp.sq_vbase = vmem;
1010         nesdev->cqp.sq_pbase = pmem;
1011         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1012         nesdev->cqp.sq_head = 0;
1013         nesdev->cqp.sq_tail = 0;
1014         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1015
1016         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1017         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1018
1019         nesdev->ccq.cq_vbase = vmem;
1020         nesdev->ccq.cq_pbase = pmem;
1021         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1022         nesdev->ccq.cq_head = 0;
1023         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1024         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1025
1026         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1027         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1028
1029         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1030         ceq = &nesadapter->ceq[nesdev->ceq_index];
1031         ceq->ceq_vbase = vmem;
1032         ceq->ceq_pbase = pmem;
1033         ceq->ceq_size = NES_CCEQ_SIZE;
1034         ceq->ceq_head = 0;
1035
1036         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1037         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1038
1039         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1040         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1041         nic_ceq->ceq_vbase = vmem;
1042         nic_ceq->ceq_pbase = pmem;
1043         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1044         nic_ceq->ceq_head = 0;
1045
1046         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1047         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1048
1049         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1050         aeq->aeq_vbase = vmem;
1051         aeq->aeq_pbase = pmem;
1052         aeq->aeq_size = nesadapter->max_qp;
1053         aeq->aeq_head = 0;
1054
1055         /* Setup QP Context */
1056         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1057         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1058
1059         cqp_qp_context = vmem;
1060         cqp_qp_context->context_words[0] =
1061                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1062         cqp_qp_context->context_words[1] = 0;
1063         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1064         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1065
1066
1067         /* Write the address to Create CQP */
1068         if ((sizeof(dma_addr_t) > 4)) {
1069                 nes_write_indexed(nesdev,
1070                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1071                                 ((u64)pmem) >> 32);
1072         } else {
1073                 nes_write_indexed(nesdev,
1074                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1075         }
1076         nes_write_indexed(nesdev,
1077                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1078                         (u32)pmem);
1079
1080         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1081         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1082
1083         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1084                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1085                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1086         }
1087
1088         /* Write Create CCQ WQE */
1089         cqp_head = nesdev->cqp.sq_head++;
1090         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1091         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1092         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1093                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1094                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1095         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1096                             (nesdev->ccq.cq_number |
1097                              ((u32)nesdev->ceq_index << 16)));
1098         u64temp = (u64)nesdev->ccq.cq_pbase;
1099         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1100         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1101         u64temp = (unsigned long)&nesdev->ccq;
1102         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1103                         cpu_to_le32((u32)(u64temp >> 1));
1104         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1105                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1106         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1107
1108         /* Write Create CEQ WQE */
1109         cqp_head = nesdev->cqp.sq_head++;
1110         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1111         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1112         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1113                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1114         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1115         u64temp = (u64)ceq->ceq_pbase;
1116         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1117
1118         /* Write Create AEQ WQE */
1119         cqp_head = nesdev->cqp.sq_head++;
1120         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1121         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1122         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1123                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1124         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1125         u64temp = (u64)aeq->aeq_pbase;
1126         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1127
1128         /* Write Create NIC CEQ WQE */
1129         cqp_head = nesdev->cqp.sq_head++;
1130         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1131         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1132         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1133                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1134         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1135         u64temp = (u64)nic_ceq->ceq_pbase;
1136         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1137
1138         /* Poll until CCQP done */
1139         count = 0;
1140         do {
1141                 if (count++ > 1000) {
1142                         printk(KERN_ERR PFX "Error creating CQP\n");
1143                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1144                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1145                         return -1;
1146                 }
1147                 udelay(10);
1148         } while (!(nes_read_indexed(nesdev,
1149                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1150
1151         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1152                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1153
1154         u32temp = 0x04800000;
1155         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1156
1157         /* wait for the CCQ, CEQ, and AEQ to get created */
1158         count = 0;
1159         do {
1160                 if (count++ > 1000) {
1161                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1162                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1163                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1164                         return -1;
1165                 }
1166                 udelay(10);
1167         } while (((nes_read_indexed(nesdev,
1168                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1169
1170         /* dump the QP status value */
1171         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1172                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1173
1174         nesdev->cqp.sq_tail++;
1175
1176         return 0;
1177 }
1178
1179
1180 /**
1181  * nes_destroy_cqp
1182  */
1183 int nes_destroy_cqp(struct nes_device *nesdev)
1184 {
1185         struct nes_hw_cqp_wqe *cqp_wqe;
1186         u32 count = 0;
1187         u32 cqp_head;
1188         unsigned long flags;
1189
1190         do {
1191                 if (count++ > 1000)
1192                         break;
1193                 udelay(10);
1194         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1195
1196         /* Reset CCQ */
1197         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1198                         nesdev->ccq.cq_number);
1199
1200         /* Disable device interrupts */
1201         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1202
1203         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1204
1205         /* Destroy the AEQ */
1206         cqp_head = nesdev->cqp.sq_head++;
1207         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1208         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1209         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1210                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1211         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1212
1213         /* Destroy the NIC CEQ */
1214         cqp_head = nesdev->cqp.sq_head++;
1215         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1216         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1217         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1218                         ((u32)nesdev->nic_ceq_index << 8));
1219
1220         /* Destroy the CEQ */
1221         cqp_head = nesdev->cqp.sq_head++;
1222         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1223         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1224         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1225                         (nesdev->ceq_index << 8));
1226
1227         /* Destroy the CCQ */
1228         cqp_head = nesdev->cqp.sq_head++;
1229         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1230         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1231         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1232         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1233                         ((u32)nesdev->ceq_index << 16));
1234
1235         /* Destroy CQP */
1236         cqp_head = nesdev->cqp.sq_head++;
1237         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1238         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1239         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1240                         NES_CQP_QP_TYPE_CQP);
1241         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1242
1243         barrier();
1244         /* Ring doorbell (5 WQEs) */
1245         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1246
1247         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1248
1249         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1250         count = 0;
1251         do {
1252                 if (count++ > 1000) {
1253                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1254                                         PCI_FUNC(nesdev->pcidev->devfn));
1255                         break;
1256                 }
1257                 udelay(10);
1258         } while (((nes_read_indexed(nesdev,
1259                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1260
1261         /* dump the QP status value */
1262         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1263                         PCI_FUNC(nesdev->pcidev->devfn),
1264                         nes_read_indexed(nesdev,
1265                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1266
1267         kfree(nesdev->nes_cqp_requests);
1268
1269         /* Free the control structures */
1270         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1271                         nesdev->cqp.sq_pbase);
1272
1273         return 0;
1274 }
1275
1276
1277 /**
1278  * nes_init_phy
1279  */
1280 int nes_init_phy(struct nes_device *nesdev)
1281 {
1282         struct nes_adapter *nesadapter = nesdev->nesadapter;
1283         u32 counter = 0;
1284         u32 sds;
1285         u32 mac_index = nesdev->mac_index;
1286         u32 tx_config = 0;
1287         u16 phy_data;
1288         u32 temp_phy_data = 0;
1289         u32 temp_phy_data2 = 0;
1290         u8  phy_type = nesadapter->phy_type[mac_index];
1291         u8  phy_index = nesadapter->phy_index[mac_index];
1292
1293         if ((nesadapter->OneG_Mode) &&
1294             (phy_type != NES_PHY_TYPE_PUMA_1G)) {
1295                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1296                 if (phy_type == NES_PHY_TYPE_1G) {
1297                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1298                         tx_config &= 0xFFFFFFE3;
1299                         tx_config |= 0x04;
1300                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1301                 }
1302
1303                 nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1304                 nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1305
1306                 /* Reset the PHY */
1307                 nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1308                 udelay(100);
1309                 counter = 0;
1310                 do {
1311                         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1312                         if (counter++ > 100)
1313                                 break;
1314                 } while (phy_data & 0x8000);
1315
1316                 /* Setting no phy loopback */
1317                 phy_data &= 0xbfff;
1318                 phy_data |= 0x1140;
1319                 nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1320                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1321                 nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1322                 nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1323
1324                 /* Setting the interrupt mask */
1325                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1326                 nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1327                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1328
1329                 /* turning on flow control */
1330                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1331                 nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1332                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1333
1334                 /* Clear Half duplex */
1335                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1336                 nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1337                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1338
1339                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1340                 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1341
1342                 return 0;
1343         }
1344
1345         if ((phy_type == NES_PHY_TYPE_IRIS) ||
1346             (phy_type == NES_PHY_TYPE_ARGUS) ||
1347             (phy_type == NES_PHY_TYPE_SFP_D)) {
1348                 /* setup 10G MDIO operation */
1349                 tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1350                 tx_config &= 0xFFFFFFE3;
1351                 tx_config |= 0x15;
1352                 nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1353         }
1354         if ((phy_type == NES_PHY_TYPE_ARGUS) ||
1355             (phy_type == NES_PHY_TYPE_SFP_D)) {
1356                 /* Check firmware heartbeat */
1357                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1358                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1359                 udelay(1500);
1360                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1361                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1362
1363                 if (temp_phy_data != temp_phy_data2)
1364                         return 0;
1365
1366                 /* no heartbeat, configure the PHY */
1367                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1368                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1369                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1370                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1371                 if (phy_type == NES_PHY_TYPE_ARGUS) {
1372                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1373                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1374                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1375                 } else {
1376                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1377                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1378                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1379                 }
1380                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1381                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1382
1383                 /* setup LEDs */
1384                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1385                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1386                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1387
1388                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1389
1390                 /* Bring PHY out of reset */
1391                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1392
1393                 /* Check for heartbeat */
1394                 counter = 0;
1395                 mdelay(690);
1396                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1397                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1398                 do {
1399                         if (counter++ > 150) {
1400                                 nes_debug(NES_DBG_PHY, "No PHY heartbeat\n");
1401                                 break;
1402                         }
1403                         mdelay(1);
1404                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1405                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1406                 } while ((temp_phy_data2 == temp_phy_data));
1407
1408                 /* wait for tracking */
1409                 counter = 0;
1410                 do {
1411                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1412                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1413                         if (counter++ > 300) {
1414                                 nes_debug(NES_DBG_PHY, "PHY did not track\n");
1415                                 break;
1416                         }
1417                         mdelay(10);
1418                 } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
1419
1420                 /* setup signal integrity */
1421                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1422                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1423                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1424                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1425                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1426
1427                 /* reset serdes */
1428                 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1429                                        mac_index * 0x200);
1430                 sds |= 0x1;
1431                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1432                                   mac_index * 0x200, sds);
1433                 sds &= 0xfffffffe;
1434                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1435                                   mac_index * 0x200, sds);
1436
1437                 counter = 0;
1438                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1439                                 && (counter++ < 5000))
1440                         ;
1441         }
1442         return 0;
1443 }
1444
1445
1446 /**
1447  * nes_replenish_nic_rq
1448  */
1449 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1450 {
1451         unsigned long flags;
1452         dma_addr_t bus_address;
1453         struct sk_buff *skb;
1454         struct nes_hw_nic_rq_wqe *nic_rqe;
1455         struct nes_hw_nic *nesnic;
1456         struct nes_device *nesdev;
1457         u32 rx_wqes_posted = 0;
1458
1459         nesnic = &nesvnic->nic;
1460         nesdev = nesvnic->nesdev;
1461         spin_lock_irqsave(&nesnic->rq_lock, flags);
1462         if (nesnic->replenishing_rq !=0) {
1463                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1464                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1465                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1466                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1467                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1468                         add_timer(&nesvnic->rq_wqes_timer);
1469                 } else
1470                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1471                 return;
1472         }
1473         nesnic->replenishing_rq = 1;
1474         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1475         do {
1476                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1477                 if (skb) {
1478                         skb->dev = nesvnic->netdev;
1479
1480                         bus_address = pci_map_single(nesdev->pcidev,
1481                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1482
1483                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1484                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1485                                         cpu_to_le32(nesvnic->max_frame_size);
1486                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1487                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1488                                         cpu_to_le32((u32)bus_address);
1489                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1490                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1491                         nesnic->rx_skb[nesnic->rq_head] = skb;
1492                         nesnic->rq_head++;
1493                         nesnic->rq_head &= nesnic->rq_size - 1;
1494                         atomic_dec(&nesvnic->rx_skbs_needed);
1495                         barrier();
1496                         if (++rx_wqes_posted == 255) {
1497                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1498                                 rx_wqes_posted = 0;
1499                         }
1500                 } else {
1501                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1502                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1503                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1504                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1505                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1506                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1507                                 add_timer(&nesvnic->rq_wqes_timer);
1508                         } else
1509                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1510                         break;
1511                 }
1512         } while (atomic_read(&nesvnic->rx_skbs_needed));
1513         barrier();
1514         if (rx_wqes_posted)
1515                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1516         nesnic->replenishing_rq = 0;
1517 }
1518
1519
1520 /**
1521  * nes_rq_wqes_timeout
1522  */
1523 static void nes_rq_wqes_timeout(unsigned long parm)
1524 {
1525         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1526         printk("%s: Timer fired.\n", __func__);
1527         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1528         if (atomic_read(&nesvnic->rx_skbs_needed))
1529                 nes_replenish_nic_rq(nesvnic);
1530 }
1531
1532
1533 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1534                                void **tcph, u64 *hdr_flags, void *priv)
1535 {
1536         unsigned int ip_len;
1537         struct iphdr *iph;
1538         skb_reset_network_header(skb);
1539         iph = ip_hdr(skb);
1540         if (iph->protocol != IPPROTO_TCP)
1541                 return -1;
1542         ip_len = ip_hdrlen(skb);
1543         skb_set_transport_header(skb, ip_len);
1544         *tcph = tcp_hdr(skb);
1545
1546         *hdr_flags = LRO_IPV4 | LRO_TCP;
1547         *iphdr = iph;
1548         return 0;
1549 }
1550
1551
1552 /**
1553  * nes_init_nic_qp
1554  */
1555 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1556 {
1557         struct nes_hw_cqp_wqe *cqp_wqe;
1558         struct nes_hw_nic_sq_wqe *nic_sqe;
1559         struct nes_hw_nic_qp_context *nic_context;
1560         struct sk_buff *skb;
1561         struct nes_hw_nic_rq_wqe *nic_rqe;
1562         struct nes_vnic *nesvnic = netdev_priv(netdev);
1563         unsigned long flags;
1564         void *vmem;
1565         dma_addr_t pmem;
1566         u64 u64temp;
1567         int ret;
1568         u32 cqp_head;
1569         u32 counter;
1570         u32 wqe_count;
1571         u8 jumbomode=0;
1572
1573         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1574         nesvnic->nic_mem_size = 256 +
1575                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1576                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1577                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1578                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1579                         sizeof(struct nes_hw_nic_qp_context);
1580
1581         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1582                         &nesvnic->nic_pbase);
1583         if (!nesvnic->nic_vbase) {
1584                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1585                 return -ENOMEM;
1586         }
1587         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1588         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1589                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1590
1591         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1592                         ~(unsigned long)(256 - 1));
1593         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1594                         ~(unsigned long long)(256 - 1));
1595
1596         /* Setup the first Fragment buffers */
1597         nesvnic->nic.first_frag_vbase = vmem;
1598
1599         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1600                 nesvnic->nic.frag_paddr[counter] = pmem;
1601                 pmem += sizeof(struct nes_first_frag);
1602         }
1603
1604         /* setup the SQ */
1605         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1606
1607         nesvnic->nic.sq_vbase = (void *)vmem;
1608         nesvnic->nic.sq_pbase = pmem;
1609         nesvnic->nic.sq_head = 0;
1610         nesvnic->nic.sq_tail = 0;
1611         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1612         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1613                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1614                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1615                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1616                                 NES_NIC_SQ_WQE_COMPLETION);
1617                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1618                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1619                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1620                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1621                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1622                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1623         }
1624
1625         nesvnic->get_cqp_request = nes_get_cqp_request;
1626         nesvnic->post_cqp_request = nes_post_cqp_request;
1627         nesvnic->mcrq_mcast_filter = NULL;
1628
1629         spin_lock_init(&nesvnic->nic.rq_lock);
1630
1631         /* setup the RQ */
1632         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1633         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1634
1635
1636         nesvnic->nic.rq_vbase = vmem;
1637         nesvnic->nic.rq_pbase = pmem;
1638         nesvnic->nic.rq_head = 0;
1639         nesvnic->nic.rq_tail = 0;
1640         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1641
1642         /* setup the CQ */
1643         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1644         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1645
1646         if (nesdev->nesadapter->netdev_count > 2)
1647                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1648         else
1649                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1650
1651         nesvnic->nic_cq.cq_vbase = vmem;
1652         nesvnic->nic_cq.cq_pbase = pmem;
1653         nesvnic->nic_cq.cq_head = 0;
1654         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1655
1656         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1657
1658         /* Send CreateCQ request to CQP */
1659         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1660         cqp_head = nesdev->cqp.sq_head;
1661
1662         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1663         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1664
1665         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1666                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1667                         ((u32)nesvnic->nic_cq.cq_size << 16));
1668         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1669                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1670         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1671         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1672         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1673         u64temp = (unsigned long)&nesvnic->nic_cq;
1674         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1675         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1676                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1677         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1678         if (++cqp_head >= nesdev->cqp.sq_size)
1679                 cqp_head = 0;
1680         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1681         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1682
1683         /* Send CreateQP request to CQP */
1684         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1685         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1686                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1687                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1688         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1689                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1690                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1691         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1692                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1693         }
1694
1695         u64temp = (u64)nesvnic->nic.sq_pbase;
1696         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1697         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1698         u64temp = (u64)nesvnic->nic.rq_pbase;
1699         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1700         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1701
1702         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1703                         NES_CQP_QP_TYPE_NIC);
1704         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1705         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1706                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1707         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1708
1709         if (++cqp_head >= nesdev->cqp.sq_size)
1710                 cqp_head = 0;
1711         nesdev->cqp.sq_head = cqp_head;
1712
1713         barrier();
1714
1715         /* Ring doorbell (2 WQEs) */
1716         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1717
1718         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1719         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1720                         nesvnic->nic.qp_id);
1721
1722         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1723                         NES_EVENT_TIMEOUT);
1724         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1725                         nesvnic->nic.qp_id, ret);
1726         if (!ret) {
1727                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1728                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1729                                 nesvnic->nic_pbase);
1730                 return -EIO;
1731         }
1732
1733         /* Populate the RQ */
1734         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1735                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1736                 if (!skb) {
1737                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1738
1739                         nes_destroy_nic_qp(nesvnic);
1740                         return -ENOMEM;
1741                 }
1742
1743                 skb->dev = netdev;
1744
1745                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1746                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1747
1748                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1749                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1750                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1751                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1752                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1753                 nesvnic->nic.rx_skb[counter] = skb;
1754         }
1755
1756         wqe_count = NES_NIC_WQ_SIZE - 1;
1757         nesvnic->nic.rq_head = wqe_count;
1758         barrier();
1759         do {
1760                 counter = min(wqe_count, ((u32)255));
1761                 wqe_count -= counter;
1762                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1763         } while (wqe_count);
1764         init_timer(&nesvnic->rq_wqes_timer);
1765         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1766         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1767         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1768         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1769         {
1770                 nes_nic_init_timer(nesdev);
1771                 if (netdev->mtu > 1500)
1772                         jumbomode = 1;
1773                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1774         }
1775         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1776         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1777         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1778         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1779         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1780         nesvnic->lro_mgr.dev            = netdev;
1781         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1782         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1783         return 0;
1784 }
1785
1786
1787 /**
1788  * nes_destroy_nic_qp
1789  */
1790 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1791 {
1792         u64 u64temp;
1793         dma_addr_t bus_address;
1794         struct nes_device *nesdev = nesvnic->nesdev;
1795         struct nes_hw_cqp_wqe *cqp_wqe;
1796         struct nes_hw_nic_sq_wqe *nic_sqe;
1797         struct nes_hw_nic_rq_wqe *nic_rqe;
1798         __le16 *wqe_fragment_length;
1799         u16  wqe_fragment_index;
1800         u64 wqe_frag;
1801         u32 cqp_head;
1802         unsigned long flags;
1803         int ret;
1804
1805         /* Free remaining NIC receive buffers */
1806         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1807                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1808                 wqe_frag  = (u64)le32_to_cpu(
1809                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1810                 wqe_frag |= ((u64)le32_to_cpu(
1811                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1812                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1813                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1814                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1815                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1816         }
1817
1818         /* Free remaining NIC transmit buffers */
1819         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1820                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1821                 wqe_fragment_index = 1;
1822                 wqe_fragment_length = (__le16 *)
1823                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1824                 /* bump past the vlan tag */
1825                 wqe_fragment_length++;
1826                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1827                         u64temp = (u64)le32_to_cpu(
1828                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1829                                 wqe_fragment_index*2]);
1830                         u64temp += ((u64)le32_to_cpu(
1831                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1832                                 + wqe_fragment_index*2]))<<32;
1833                         bus_address = (dma_addr_t)u64temp;
1834                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1835                                         nesvnic->nic.first_frag_overflow)) {
1836                                 pci_unmap_single(nesdev->pcidev,
1837                                                 bus_address,
1838                                                 le16_to_cpu(wqe_fragment_length[
1839                                                         wqe_fragment_index++]),
1840                                                 PCI_DMA_TODEVICE);
1841                         }
1842                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1843                                 if (wqe_fragment_length[wqe_fragment_index]) {
1844                                         u64temp = le32_to_cpu(
1845                                                 nic_sqe->wqe_words[
1846                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1847                                                 wqe_fragment_index*2]);
1848                                         u64temp += ((u64)le32_to_cpu(
1849                                                 nic_sqe->wqe_words[
1850                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1851                                                 wqe_fragment_index*2]))<<32;
1852                                         bus_address = (dma_addr_t)u64temp;
1853                                         pci_unmap_page(nesdev->pcidev,
1854                                                         bus_address,
1855                                                         le16_to_cpu(
1856                                                         wqe_fragment_length[
1857                                                         wqe_fragment_index]),
1858                                                         PCI_DMA_TODEVICE);
1859                                 } else
1860                                         break;
1861                         }
1862                 }
1863                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1864                         dev_kfree_skb(
1865                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1866
1867                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1868                                         & (nesvnic->nic.sq_size - 1);
1869         }
1870
1871         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1872
1873         /* Destroy NIC QP */
1874         cqp_head = nesdev->cqp.sq_head;
1875         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1876         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1877
1878         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1879                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1880         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1881                 nesvnic->nic.qp_id);
1882
1883         if (++cqp_head >= nesdev->cqp.sq_size)
1884                 cqp_head = 0;
1885
1886         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1887
1888         /* Destroy NIC CQ */
1889         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1890         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1891                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1892         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1893                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1894
1895         if (++cqp_head >= nesdev->cqp.sq_size)
1896                 cqp_head = 0;
1897
1898         nesdev->cqp.sq_head = cqp_head;
1899         barrier();
1900
1901         /* Ring doorbell (2 WQEs) */
1902         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1903
1904         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1905         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1906                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1907                         cqp_head, nesdev->cqp.sq_head,
1908                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1909
1910         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1911                         NES_EVENT_TIMEOUT);
1912
1913         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1914                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1915                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1916         if (!ret) {
1917                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1918                                 nesvnic->nic.qp_id);
1919         }
1920
1921         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1922                         nesvnic->nic_pbase);
1923 }
1924
1925 /**
1926  * nes_napi_isr
1927  */
1928 int nes_napi_isr(struct nes_device *nesdev)
1929 {
1930         struct nes_adapter *nesadapter = nesdev->nesadapter;
1931         u32 int_stat;
1932
1933         if (nesdev->napi_isr_ran) {
1934                 /* interrupt status has already been read in ISR */
1935                 int_stat = nesdev->int_stat;
1936         } else {
1937                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1938                 nesdev->int_stat = int_stat;
1939                 nesdev->napi_isr_ran = 1;
1940         }
1941
1942         int_stat &= nesdev->int_req;
1943         /* iff NIC, process here, else wait for DPC */
1944         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1945                 nesdev->napi_isr_ran = 0;
1946                 nes_write32(nesdev->regs + NES_INT_STAT,
1947                         (int_stat &
1948                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1949
1950                 /* Process the CEQs */
1951                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1952
1953                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1954                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1955                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1956                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1957                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1958                                 /* Enable Periodic timer interrupts */
1959                                 nesdev->int_req |= NES_INT_TIMER;
1960                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1961                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1962                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1963                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1964                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1965                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1966                         }
1967
1968                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1969                         {
1970                                 nes_nic_init_timer(nesdev);
1971                         }
1972                         /* Enable interrupts, except CEQs */
1973                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1974                 } else {
1975                         /* Enable interrupts, make sure timer is off */
1976                         nesdev->int_req &= ~NES_INT_TIMER;
1977                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1978                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1979                 }
1980                 nesdev->deepcq_count = 0;
1981                 return 1;
1982         } else {
1983                 return 0;
1984         }
1985 }
1986
1987 static void process_critical_error(struct nes_device *nesdev)
1988 {
1989         u32 debug_error;
1990         u32 nes_idx_debug_error_masks0 = 0;
1991         u16 error_module = 0;
1992
1993         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1994         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1995                         (u16)debug_error);
1996         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1997                         0x01010000 | (debug_error & 0x0000ffff));
1998         if (crit_err_count++ > 10)
1999                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2000         error_module = (u16) (debug_error & 0x1F00) >> 8;
2001         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2002                         nes_max_critical_error_count) {
2003                 printk(KERN_ERR PFX "Masking off critical error for module "
2004                         "0x%02X\n", (u16)error_module);
2005                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2006                         NES_IDX_DEBUG_ERROR_MASKS0);
2007                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2008                         nes_idx_debug_error_masks0 | (1 << error_module));
2009         }
2010 }
2011 /**
2012  * nes_dpc
2013  */
2014 void nes_dpc(unsigned long param)
2015 {
2016         struct nes_device *nesdev = (struct nes_device *)param;
2017         struct nes_adapter *nesadapter = nesdev->nesadapter;
2018         u32 counter;
2019         u32 loop_counter = 0;
2020         u32 int_status_bit;
2021         u32 int_stat;
2022         u32 timer_stat;
2023         u32 temp_int_stat;
2024         u32 intf_int_stat;
2025         u32 processed_intf_int = 0;
2026         u16 processed_timer_int = 0;
2027         u16 completion_ints = 0;
2028         u16 timer_ints = 0;
2029
2030         /* nes_debug(NES_DBG_ISR, "\n"); */
2031
2032         do {
2033                 timer_stat = 0;
2034                 if (nesdev->napi_isr_ran) {
2035                         nesdev->napi_isr_ran = 0;
2036                         int_stat = nesdev->int_stat;
2037                 } else
2038                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2039                 if (processed_intf_int != 0)
2040                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2041                 else
2042                         int_stat &= nesdev->int_req;
2043                 if (processed_timer_int == 0) {
2044                         processed_timer_int = 1;
2045                         if (int_stat & NES_INT_TIMER) {
2046                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2047                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2048                                         int_stat &= ~NES_INT_TIMER;
2049                                 }
2050                         }
2051                 } else {
2052                         int_stat &= ~NES_INT_TIMER;
2053                 }
2054
2055                 if (int_stat) {
2056                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2057                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2058                                 /* Ack the interrupts */
2059                                 nes_write32(nesdev->regs+NES_INT_STAT,
2060                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2061                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2062                         }
2063
2064                         temp_int_stat = int_stat;
2065                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2066                                 if (int_stat & int_status_bit) {
2067                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2068                                         temp_int_stat &= ~int_status_bit;
2069                                         completion_ints = 1;
2070                                 }
2071                                 if (!(temp_int_stat & 0x0000ffff))
2072                                         break;
2073                                 int_status_bit <<= 1;
2074                         }
2075
2076                         /* Process the AEQ for this pci function */
2077                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2078                         if (int_stat & int_status_bit) {
2079                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2080                         }
2081
2082                         /* Process the MAC interrupt for this pci function */
2083                         int_status_bit = 1 << (24 + nesdev->mac_index);
2084                         if (int_stat & int_status_bit) {
2085                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2086                         }
2087
2088                         if (int_stat & NES_INT_TIMER) {
2089                                 if (timer_stat & nesdev->timer_int_req) {
2090                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2091                                                         (timer_stat & nesdev->timer_int_req) |
2092                                                         ~(nesdev->nesadapter->timer_int_req));
2093                                         timer_ints = 1;
2094                                 }
2095                         }
2096
2097                         if (int_stat & NES_INT_INTF) {
2098                                 processed_intf_int = 1;
2099                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2100                                 intf_int_stat &= nesdev->intf_int_req;
2101                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2102                                         process_critical_error(nesdev);
2103                                 }
2104                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2105                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2106                                         BUG();
2107                                 }
2108                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2109                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2110                                         BUG();
2111                                 }
2112                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2113                         }
2114
2115                         if (int_stat & NES_INT_TSW) {
2116                         }
2117                 }
2118                 /* Don't use the interface interrupt bit stay in loop */
2119                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2120                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2121         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2122
2123         if (timer_ints == 1) {
2124                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2125                         if (completion_ints == 0) {
2126                                 nesdev->timer_only_int_count++;
2127                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2128                                         nesdev->timer_only_int_count = 0;
2129                                         nesdev->int_req &= ~NES_INT_TIMER;
2130                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2131                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2132                                 } else {
2133                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2134                                 }
2135                         } else {
2136                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2137                                 {
2138                                         nes_nic_init_timer(nesdev);
2139                                 }
2140                                 nesdev->timer_only_int_count = 0;
2141                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2142                         }
2143                 } else {
2144                         nesdev->timer_only_int_count = 0;
2145                         nesdev->int_req &= ~NES_INT_TIMER;
2146                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2147                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2148                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2149                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2150                 }
2151         } else {
2152                 if ( (completion_ints == 1) &&
2153                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2154                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2155                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2156                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2157                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2158                         nesdev->timer_only_int_count = 0;
2159                         nesdev->int_req |= NES_INT_TIMER;
2160                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2161                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2162                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2163                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2164                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2165                 } else {
2166                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2167                 }
2168         }
2169         nesdev->deepcq_count = 0;
2170 }
2171
2172
2173 /**
2174  * nes_process_ceq
2175  */
2176 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2177 {
2178         u64 u64temp;
2179         struct nes_hw_cq *cq;
2180         u32 head;
2181         u32 ceq_size;
2182
2183         /* nes_debug(NES_DBG_CQ, "\n"); */
2184         head = ceq->ceq_head;
2185         ceq_size = ceq->ceq_size;
2186
2187         do {
2188                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2189                                 NES_CEQE_VALID) {
2190                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2191                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2192                         u64temp <<= 1;
2193                         cq = *((struct nes_hw_cq **)&u64temp);
2194                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2195                         barrier();
2196                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2197
2198                         /* call the event handler */
2199                         cq->ce_handler(nesdev, cq);
2200
2201                         if (++head >= ceq_size)
2202                                 head = 0;
2203                 } else {
2204                         break;
2205                 }
2206
2207         } while (1);
2208
2209         ceq->ceq_head = head;
2210 }
2211
2212
2213 /**
2214  * nes_process_aeq
2215  */
2216 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2217 {
2218         /* u64 u64temp; */
2219         u32 head;
2220         u32 aeq_size;
2221         u32 aeqe_misc;
2222         u32 aeqe_cq_id;
2223         struct nes_hw_aeqe volatile *aeqe;
2224
2225         head = aeq->aeq_head;
2226         aeq_size = aeq->aeq_size;
2227
2228         do {
2229                 aeqe = &aeq->aeq_vbase[head];
2230                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2231                         break;
2232                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2233                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2234                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2235                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2236                                 /* dealing with an accelerated QP related AE */
2237                                 /*
2238                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2239                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2240                                  */
2241                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2242                         } else {
2243                                 /* TODO: dealing with a CQP related AE */
2244                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2245                                                 (u16)(aeqe_misc >> 16));
2246                         }
2247                 }
2248
2249                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2250
2251                 if (++head >= aeq_size)
2252                         head = 0;
2253
2254                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2255         }
2256         while (1);
2257         aeq->aeq_head = head;
2258 }
2259
2260 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2261 {
2262         struct nes_adapter *nesadapter = nesdev->nesadapter;
2263         u32 reset_value;
2264         u32 i=0;
2265         u32 u32temp;
2266
2267         if (nesadapter->hw_rev == NE020_REV) {
2268                 return;
2269         }
2270         mh_detected++;
2271
2272         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2273
2274         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2275                 reset_value |= 0x0000001d;
2276         else
2277                 reset_value |= 0x0000002d;
2278
2279         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2280                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2281                         nesadapter->link_interrupt_count[0] = 0;
2282                         nesadapter->link_interrupt_count[1] = 0;
2283                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2284                         if (0x00000040 & u32temp)
2285                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2286                         else
2287                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2288
2289                         reset_value |= 0x0000003d;
2290                 }
2291                 nesadapter->link_interrupt_count[mac_index] = 0;
2292         }
2293
2294         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2295
2296         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2297                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2298
2299         if (0x0000003d == (reset_value & 0x0000003d)) {
2300                 u32 pcs_control_status0, pcs_control_status1;
2301
2302                 for (i = 0; i < 10; i++) {
2303                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2304                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2305                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2306                              && (pcs_control_status0 & 0x00100000))
2307                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2308                                 && (pcs_control_status1 & 0x00100000)))
2309                                 continue;
2310                         else
2311                                 break;
2312                 }
2313                 if (10 == i) {
2314                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2315                         if (0x00000040 & u32temp)
2316                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2317                         else
2318                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2319
2320                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2321
2322                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2323                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2324                 }
2325         }
2326 }
2327
2328 /**
2329  * nes_process_mac_intr
2330  */
2331 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2332 {
2333         unsigned long flags;
2334         u32 pcs_control_status;
2335         struct nes_adapter *nesadapter = nesdev->nesadapter;
2336         struct nes_vnic *nesvnic;
2337         u32 mac_status;
2338         u32 mac_index = nesdev->mac_index;
2339         u32 u32temp;
2340         u16 phy_data;
2341         u16 temp_phy_data;
2342         u32 pcs_val  = 0x0f0f0000;
2343         u32 pcs_mask = 0x0f1f0000;
2344         u32 cdr_ctrl;
2345
2346         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2347         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2348                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2349                 return;
2350         }
2351         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2352         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2353
2354         /* ack the MAC interrupt */
2355         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2356         /* Clear the interrupt */
2357         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2358
2359         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2360
2361         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2362                 nesdev->link_status_interrupts++;
2363                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2364                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2365                         nes_reset_link(nesdev, mac_index);
2366                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2367                 }
2368                 /* read the PHY interrupt status register */
2369                 if ((nesadapter->OneG_Mode) &&
2370                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2371                         do {
2372                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2373                                                 nesadapter->phy_index[mac_index], &phy_data);
2374                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2375                                                 nesadapter->phy_index[mac_index], phy_data);
2376                         } while (phy_data&0x8000);
2377
2378                         temp_phy_data = 0;
2379                         do {
2380                                 nes_read_1G_phy_reg(nesdev, 0x11,
2381                                                 nesadapter->phy_index[mac_index], &phy_data);
2382                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2383                                                 nesadapter->phy_index[mac_index], phy_data);
2384                                 if (temp_phy_data == phy_data)
2385                                         break;
2386                                 temp_phy_data = phy_data;
2387                         } while (1);
2388
2389                         nes_read_1G_phy_reg(nesdev, 0x1e,
2390                                         nesadapter->phy_index[mac_index], &phy_data);
2391                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2392                                         nesadapter->phy_index[mac_index], phy_data);
2393
2394                         nes_read_1G_phy_reg(nesdev, 1,
2395                                         nesadapter->phy_index[mac_index], &phy_data);
2396                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2397                                         nesadapter->phy_index[mac_index], phy_data);
2398
2399                         if (temp_phy_data & 0x1000) {
2400                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2401                                 phy_data = 4;
2402                         } else {
2403                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2404                         }
2405                 }
2406                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2407                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2408                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2409
2410                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2411                         switch (mac_index) {
2412                         case 1:
2413                         case 3:
2414                                 pcs_control_status = nes_read_indexed(nesdev,
2415                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2416                                 break;
2417                         default:
2418                                 pcs_control_status = nes_read_indexed(nesdev,
2419                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2420                                 break;
2421                         }
2422                 } else {
2423                         pcs_control_status = nes_read_indexed(nesdev,
2424                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2425                         pcs_control_status = nes_read_indexed(nesdev,
2426                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2427                 }
2428
2429                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2430                                 mac_index, pcs_control_status);
2431                 if ((nesadapter->OneG_Mode) &&
2432                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2433                         u32temp = 0x01010000;
2434                         if (nesadapter->port_count > 2) {
2435                                 u32temp |= 0x02020000;
2436                         }
2437                         if ((pcs_control_status & u32temp)!= u32temp) {
2438                                 phy_data = 0;
2439                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2440                         }
2441                 } else {
2442                         switch (nesadapter->phy_type[mac_index]) {
2443                         case NES_PHY_TYPE_IRIS:
2444                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2445                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2446                                 u32temp = 20;
2447                                 do {
2448                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2449                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2450                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2451                                                 break;
2452                                         temp_phy_data = phy_data;
2453                                 } while (1);
2454                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2455                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2456                                 break;
2457
2458                         case NES_PHY_TYPE_ARGUS:
2459                         case NES_PHY_TYPE_SFP_D:
2460                                 /* clear the alarms */
2461                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2462                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2463                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2464                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2465                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2466                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2467                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2468                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2469                                 /* check link status */
2470                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2471                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2472
2473                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2474                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2475                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2476                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2477
2478                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2479
2480                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2481                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2482                                 break;
2483
2484                         case NES_PHY_TYPE_PUMA_1G:
2485                                 if (mac_index < 2)
2486                                         pcs_val = pcs_mask = 0x01010000;
2487                                 else
2488                                         pcs_val = pcs_mask = 0x02020000;
2489                                 /* fall through */
2490                         default:
2491                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2492                                 break;
2493                         }
2494                 }
2495
2496                 if (phy_data & 0x0004) {
2497                         if (wide_ppm_offset &&
2498                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2499                             (nesadapter->hw_rev != NE020_REV)) {
2500                                 cdr_ctrl = nes_read_indexed(nesdev,
2501                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2502                                                             mac_index * 0x200);
2503                                 nes_write_indexed(nesdev,
2504                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2505                                                   mac_index * 0x200,
2506                                                   cdr_ctrl | 0x000F0000);
2507                         }
2508                         nesadapter->mac_link_down[mac_index] = 0;
2509                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2510                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2511                                                 nesvnic->linkup);
2512                                 if (nesvnic->linkup == 0) {
2513                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2514                                                         nesvnic->netdev->name, nesvnic->netdev);
2515                                         if (netif_queue_stopped(nesvnic->netdev))
2516                                                 netif_start_queue(nesvnic->netdev);
2517                                         nesvnic->linkup = 1;
2518                                         netif_carrier_on(nesvnic->netdev);
2519                                 }
2520                         }
2521                 } else {
2522                         if (wide_ppm_offset &&
2523                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2524                             (nesadapter->hw_rev != NE020_REV)) {
2525                                 cdr_ctrl = nes_read_indexed(nesdev,
2526                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2527                                                             mac_index * 0x200);
2528                                 nes_write_indexed(nesdev,
2529                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2530                                                   mac_index * 0x200,
2531                                                   cdr_ctrl & 0xFFF0FFFF);
2532                         }
2533                         nesadapter->mac_link_down[mac_index] = 1;
2534                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2535                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2536                                                 nesvnic->linkup);
2537                                 if (nesvnic->linkup == 1) {
2538                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2539                                                         nesvnic->netdev->name, nesvnic->netdev);
2540                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2541                                                 netif_stop_queue(nesvnic->netdev);
2542                                         nesvnic->linkup = 0;
2543                                         netif_carrier_off(nesvnic->netdev);
2544                                 }
2545                         }
2546                 }
2547         }
2548
2549         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2550 }
2551
2552
2553
2554 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2555 {
2556         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2557
2558         napi_schedule(&nesvnic->napi);
2559 }
2560
2561
2562 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2563 * getting out of nic_ce_handler
2564 */
2565 #define MAX_RQES_TO_PROCESS     384
2566
2567 /**
2568  * nes_nic_ce_handler
2569  */
2570 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2571 {
2572         u64 u64temp;
2573         dma_addr_t bus_address;
2574         struct nes_hw_nic *nesnic;
2575         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2576         struct nes_adapter *nesadapter = nesdev->nesadapter;
2577         struct nes_hw_nic_rq_wqe *nic_rqe;
2578         struct nes_hw_nic_sq_wqe *nic_sqe;
2579         struct sk_buff *skb;
2580         struct sk_buff *rx_skb;
2581         __le16 *wqe_fragment_length;
2582         u32 head;
2583         u32 cq_size;
2584         u32 rx_pkt_size;
2585         u32 cqe_count=0;
2586         u32 cqe_errv;
2587         u32 cqe_misc;
2588         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2589         u16 vlan_tag;
2590         u16 pkt_type;
2591         u16 rqes_processed = 0;
2592         u8 sq_cqes = 0;
2593         u8 nes_use_lro = 0;
2594
2595         head = cq->cq_head;
2596         cq_size = cq->cq_size;
2597         cq->cqes_pending = 1;
2598         if (nesvnic->netdev->features & NETIF_F_LRO)
2599                 nes_use_lro = 1;
2600         do {
2601                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2602                                 NES_NIC_CQE_VALID) {
2603                         nesnic = &nesvnic->nic;
2604                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2605                         if (cqe_misc & NES_NIC_CQE_SQ) {
2606                                 sq_cqes++;
2607                                 wqe_fragment_index = 1;
2608                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2609                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2610                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2611                                 /* bump past the vlan tag */
2612                                 wqe_fragment_length++;
2613                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2614                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2615                                                         wqe_fragment_index * 2]);
2616                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2617                                                         wqe_fragment_index * 2])) << 32;
2618                                         bus_address = (dma_addr_t)u64temp;
2619                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2620                                                 pci_unmap_single(nesdev->pcidev,
2621                                                                 bus_address,
2622                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2623                                                                 PCI_DMA_TODEVICE);
2624                                         }
2625                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2626                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2627                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2628                                                                                 wqe_fragment_index * 2]);
2629                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2630                                                                                 + wqe_fragment_index * 2])) <<32;
2631                                                         bus_address = (dma_addr_t)u64temp;
2632                                                         pci_unmap_page(nesdev->pcidev,
2633                                                                         bus_address,
2634                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2635                                                                         PCI_DMA_TODEVICE);
2636                                                 } else
2637                                                         break;
2638                                         }
2639                                 }
2640                                 if (skb)
2641                                         dev_kfree_skb_any(skb);
2642                                 nesnic->sq_tail++;
2643                                 nesnic->sq_tail &= nesnic->sq_size-1;
2644                                 if (sq_cqes > 128) {
2645                                         barrier();
2646                                 /* restart the queue if it had been stopped */
2647                                 if (netif_queue_stopped(nesvnic->netdev))
2648                                         netif_wake_queue(nesvnic->netdev);
2649                                         sq_cqes = 0;
2650                                 }
2651                         } else {
2652                                 rqes_processed ++;
2653
2654                                 cq->rx_cqes_completed++;
2655                                 cq->rx_pkts_indicated++;
2656                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2657                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2658                                 /* Get the skb */
2659                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2660                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2661                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2662                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2663                                 pci_unmap_single(nesdev->pcidev, bus_address,
2664                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2665                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2666                                 /* rx_skb->len = rx_pkt_size; */
2667                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2668                                 skb_put(rx_skb, rx_pkt_size);
2669                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2670                                 nesnic->rq_tail++;
2671                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2672
2673                                 atomic_inc(&nesvnic->rx_skbs_needed);
2674                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2675                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2676                                                         cq->cq_number | (cqe_count << 16));
2677                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2678                                         nesdev->currcq_count += cqe_count;
2679                                         cqe_count = 0;
2680                                         nes_replenish_nic_rq(nesvnic);
2681                                 }
2682                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2683                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2684                                 rx_skb->ip_summed = CHECKSUM_NONE;
2685
2686                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2687                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2688                                         if ((cqe_errv &
2689                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2690                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2691                                                 if (nesvnic->rx_checksum_disabled == 0) {
2692                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2693                                                 }
2694                                         } else
2695                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2696                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2697                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2698
2699                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2700                                         if ((cqe_errv &
2701                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2702                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2703                                                 if (nesvnic->rx_checksum_disabled == 0) {
2704                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2705                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2706                                                                   nesvnic->netdev->name); */
2707                                                 }
2708                                         } else
2709                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2710                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2711                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2712                                         }
2713                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2714                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2715
2716                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2717                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2718                                                 rx_skb = NULL;
2719                                 }
2720                                 if (rx_skb == NULL)
2721                                         goto skip_rx_indicate0;
2722
2723
2724                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2725                                     (nesvnic->vlan_grp != NULL)) {
2726                                         vlan_tag = (u16)(le32_to_cpu(
2727                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2728                                                         >> 16);
2729                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2730                                                         nesvnic->netdev->name, vlan_tag);
2731                                         if (nes_use_lro)
2732                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2733                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2734                                         else
2735                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2736                                 } else {
2737                                         if (nes_use_lro)
2738                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2739                                         else
2740                                                 nes_netif_rx(rx_skb);
2741                                 }
2742
2743 skip_rx_indicate0:
2744                                 nesvnic->netdev->last_rx = jiffies;
2745                                 /* nesvnic->netstats.rx_packets++; */
2746                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2747                         }
2748
2749                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2750                         /* Accounting... */
2751                         cqe_count++;
2752                         if (++head >= cq_size)
2753                                 head = 0;
2754                         if (cqe_count == 255) {
2755                                 /* Replenish Nic CQ */
2756                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2757                                                 cq->cq_number | (cqe_count << 16));
2758                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2759                                 nesdev->currcq_count += cqe_count;
2760                                 cqe_count = 0;
2761                         }
2762
2763                         if (cq->rx_cqes_completed >= nesvnic->budget)
2764                                 break;
2765                 } else {
2766                         cq->cqes_pending = 0;
2767                         break;
2768                 }
2769
2770         } while (1);
2771
2772         if (nes_use_lro)
2773                 lro_flush_all(&nesvnic->lro_mgr);
2774         if (sq_cqes) {
2775                 barrier();
2776                 /* restart the queue if it had been stopped */
2777                 if (netif_queue_stopped(nesvnic->netdev))
2778                         netif_wake_queue(nesvnic->netdev);
2779         }
2780         cq->cq_head = head;
2781         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2782                         cq->cq_number, cqe_count, cq->cq_head); */
2783         cq->cqe_allocs_pending = cqe_count;
2784         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2785         {
2786                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2787                 nesdev->currcq_count += cqe_count;
2788                 nes_nic_tune_timer(nesdev);
2789         }
2790         if (atomic_read(&nesvnic->rx_skbs_needed))
2791                 nes_replenish_nic_rq(nesvnic);
2792 }
2793
2794
2795 /**
2796  * nes_cqp_ce_handler
2797  */
2798 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2799 {
2800         u64 u64temp;
2801         unsigned long flags;
2802         struct nes_hw_cqp *cqp = NULL;
2803         struct nes_cqp_request *cqp_request;
2804         struct nes_hw_cqp_wqe *cqp_wqe;
2805         u32 head;
2806         u32 cq_size;
2807         u32 cqe_count=0;
2808         u32 error_code;
2809         /* u32 counter; */
2810
2811         head = cq->cq_head;
2812         cq_size = cq->cq_size;
2813
2814         do {
2815                 /* process the CQE */
2816                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2817                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2818
2819                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2820                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2821                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2822                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2823                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2824                         cqp = *((struct nes_hw_cqp **)&u64temp);
2825
2826                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2827                         if (error_code) {
2828                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2829                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2830                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2831                                                 (u16)(error_code >> 16),
2832                                                 (u16)error_code);
2833                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2834                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2835                         }
2836
2837                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2838                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2839                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2840                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2841                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2842                         if (cqp_request) {
2843                                 if (cqp_request->waiting) {
2844                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2845                                         cqp_request->major_code = (u16)(error_code >> 16);
2846                                         cqp_request->minor_code = (u16)error_code;
2847                                         barrier();
2848                                         cqp_request->request_done = 1;
2849                                         wake_up(&cqp_request->waitq);
2850                                         nes_put_cqp_request(nesdev, cqp_request);
2851                                 } else {
2852                                         if (cqp_request->callback)
2853                                                 cqp_request->cqp_callback(nesdev, cqp_request);
2854                                         nes_free_cqp_request(nesdev, cqp_request);
2855                                 }
2856                         } else {
2857                                 wake_up(&nesdev->cqp.waitq);
2858                         }
2859
2860                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2861                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2862                         if (++cqp->sq_tail >= cqp->sq_size)
2863                                 cqp->sq_tail = 0;
2864
2865                         /* Accounting... */
2866                         cqe_count++;
2867                         if (++head >= cq_size)
2868                                 head = 0;
2869                 } else {
2870                         break;
2871                 }
2872         } while (1);
2873         cq->cq_head = head;
2874
2875         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2876         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2877                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2878                         (nesdev->cqp.sq_size - 1)) != 1)) {
2879                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2880                                 struct nes_cqp_request, list);
2881                 list_del_init(&cqp_request->list);
2882                 head = nesdev->cqp.sq_head++;
2883                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2884                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2885                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2886                 barrier();
2887                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2888                         cpu_to_le32((u32)((unsigned long)cqp_request));
2889                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2890                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2891                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2892                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2893                 /* Ring doorbell (1 WQEs) */
2894                 barrier();
2895                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2896         }
2897         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2898
2899         /* Arm the CCQ */
2900         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2901                         cq->cq_number);
2902         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2903 }
2904
2905
2906 /**
2907  * nes_process_iwarp_aeqe
2908  */
2909 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2910                                    struct nes_hw_aeqe *aeqe)
2911 {
2912         u64 context;
2913         u64 aeqe_context = 0;
2914         unsigned long flags;
2915         struct nes_qp *nesqp;
2916         int resource_allocated;
2917         /* struct iw_cm_id *cm_id; */
2918         struct nes_adapter *nesadapter = nesdev->nesadapter;
2919         struct ib_event ibevent;
2920         /* struct iw_cm_event cm_event; */
2921         u32 aeq_info;
2922         u32 next_iwarp_state = 0;
2923         u16 async_event_id;
2924         u8 tcp_state;
2925         u8 iwarp_state;
2926
2927         nes_debug(NES_DBG_AEQ, "\n");
2928         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2929         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2930                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2931                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2932         } else {
2933                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2934                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2935                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2936                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
2937                 BUG_ON(!context);
2938         }
2939
2940         async_event_id = (u16)aeq_info;
2941         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2942         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2943         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2944                         " Tcp state = %s, iWARP state = %s\n",
2945                         async_event_id,
2946                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2947                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2948
2949         switch (async_event_id) {
2950                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2951                         nesqp = *((struct nes_qp **)&context);
2952                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2953                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2954                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2955                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2956                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2957                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2958                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2959                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2960                                                 async_event_id, nesqp->last_aeq, tcp_state);
2961                         }
2962                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2963                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2964                                 /* FIN Received but tcp state or IB state moved on,
2965                                                 should expect a close complete */
2966                                 return;
2967                         }
2968                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2969                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2970                 case NES_AEQE_AEID_TERMINATE_SENT:
2971                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2972                 case NES_AEQE_AEID_RESET_SENT:
2973                         nesqp = *((struct nes_qp **)&context);
2974                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2975                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2976                         }
2977                         spin_lock_irqsave(&nesqp->lock, flags);
2978                         nesqp->hw_iwarp_state = iwarp_state;
2979                         nesqp->hw_tcp_state = tcp_state;
2980                         nesqp->last_aeq = async_event_id;
2981
2982                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2983                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2984                                 nesqp->hte_added = 0;
2985                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2986                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2987                                                 nesqp->hwqp.qp_id);
2988                                 nes_hw_modify_qp(nesdev, nesqp,
2989                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2990                                 spin_lock_irqsave(&nesqp->lock, flags);
2991                         }
2992
2993                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2994                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2995                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2996                                 switch (nesqp->hw_iwarp_state) {
2997                                         case NES_AEQE_IWARP_STATE_RTS:
2998                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2999                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3000                                                 break;
3001                                         case NES_AEQE_IWARP_STATE_TERMINATE:
3002                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
3003                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
3004                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
3005                                                         next_iwarp_state |= 0x02000000;
3006                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3007                                                 }
3008                                                 break;
3009                                         default:
3010                                                 next_iwarp_state = 0;
3011                                 }
3012                                 spin_unlock_irqrestore(&nesqp->lock, flags);
3013                                 if (next_iwarp_state) {
3014                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3015                                                         " also added another reference\n",
3016                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3017                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3018                                 }
3019                                 nes_cm_disconn(nesqp);
3020                         } else {
3021                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
3022                                         /* FIN Received but ib state not RTS,
3023                                                         close complete will be on its way */
3024                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3025                                         return;
3026                                 }
3027                                 spin_unlock_irqrestore(&nesqp->lock, flags);
3028                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
3029                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
3030                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3031                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3032                                                         " also added another reference\n",
3033                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3034                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3035                                 }
3036                                 nes_cm_disconn(nesqp);
3037                         }
3038                         break;
3039                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3040                         nesqp = *((struct nes_qp **)&context);
3041                         spin_lock_irqsave(&nesqp->lock, flags);
3042                         nesqp->hw_iwarp_state = iwarp_state;
3043                         nesqp->hw_tcp_state = tcp_state;
3044                         nesqp->last_aeq = async_event_id;
3045                         spin_unlock_irqrestore(&nesqp->lock, flags);
3046                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
3047                                         " event on QP%u \n  Q2 Data:\n",
3048                                         nesqp->hwqp.qp_id);
3049                         if (nesqp->ibqp.event_handler) {
3050                                 ibevent.device = nesqp->ibqp.device;
3051                                 ibevent.element.qp = &nesqp->ibqp;
3052                                 ibevent.event = IB_EVENT_QP_FATAL;
3053                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3054                         }
3055                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3056                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
3057                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
3058                                 nes_cm_disconn(nesqp);
3059                         } else {
3060                                 nesqp->in_disconnect = 0;
3061                                 wake_up(&nesqp->kick_waitq);
3062                         }
3063                         break;
3064                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3065                         nesqp = *((struct nes_qp **)&context);
3066                         spin_lock_irqsave(&nesqp->lock, flags);
3067                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
3068                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3069                         nesqp->last_aeq = async_event_id;
3070                         if (nesqp->cm_id) {
3071                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3072                                                 " event on QP%u, remote IP = 0x%08X \n",
3073                                                 nesqp->hwqp.qp_id,
3074                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
3075                         } else {
3076                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3077                                                 " event on QP%u \n",
3078                                                 nesqp->hwqp.qp_id);
3079                         }
3080                         spin_unlock_irqrestore(&nesqp->lock, flags);
3081                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
3082                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3083                         if (nesqp->ibqp.event_handler) {
3084                                 ibevent.device = nesqp->ibqp.device;
3085                                 ibevent.element.qp = &nesqp->ibqp;
3086                                 ibevent.event = IB_EVENT_QP_FATAL;
3087                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3088                         }
3089                         break;
3090                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3091                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
3092                                 nesqp = nesadapter->qp_table[le32_to_cpu(
3093                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3094                         } else {
3095                                 /* TODO: get the actual WQE and mask off wqe index */
3096                                 context &= ~((u64)511);
3097                                 nesqp = *((struct nes_qp **)&context);
3098                         }
3099                         spin_lock_irqsave(&nesqp->lock, flags);
3100                         nesqp->hw_iwarp_state = iwarp_state;
3101                         nesqp->hw_tcp_state = tcp_state;
3102                         nesqp->last_aeq = async_event_id;
3103                         spin_unlock_irqrestore(&nesqp->lock, flags);
3104                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
3105                                         nesqp->hwqp.qp_id);
3106                         if (nesqp->ibqp.event_handler) {
3107                                 ibevent.device = nesqp->ibqp.device;
3108                                 ibevent.element.qp = &nesqp->ibqp;
3109                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3110                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3111                         }
3112                         break;
3113                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3114                         nesqp = *((struct nes_qp **)&context);
3115                         spin_lock_irqsave(&nesqp->lock, flags);
3116                         nesqp->hw_iwarp_state = iwarp_state;
3117                         nesqp->hw_tcp_state = tcp_state;
3118                         nesqp->last_aeq = async_event_id;
3119                         spin_unlock_irqrestore(&nesqp->lock, flags);
3120                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
3121                                         nesqp->hwqp.qp_id);
3122                         if (nesqp->ibqp.event_handler) {
3123                                 ibevent.device = nesqp->ibqp.device;
3124                                 ibevent.element.qp = &nesqp->ibqp;
3125                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3126                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3127                         }
3128                         break;
3129                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3130                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
3131                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3132                         spin_lock_irqsave(&nesqp->lock, flags);
3133                         nesqp->hw_iwarp_state = iwarp_state;
3134                         nesqp->hw_tcp_state = tcp_state;
3135                         nesqp->last_aeq = async_event_id;
3136                         spin_unlock_irqrestore(&nesqp->lock, flags);
3137                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
3138                                         " nesqp = %p, AE reported %p\n",
3139                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
3140                         if (nesqp->ibqp.event_handler) {
3141                                 ibevent.device = nesqp->ibqp.device;
3142                                 ibevent.element.qp = &nesqp->ibqp;
3143                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3144                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3145                         }
3146                         break;
3147                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3148                         context <<= 1;
3149                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3150                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3151                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3152                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3153                         if (resource_allocated) {
3154                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3155                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3156                         }
3157                         break;
3158                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3159                         nesqp = nesadapter->qp_table[le32_to_cpu(
3160                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3161                         spin_lock_irqsave(&nesqp->lock, flags);
3162                         nesqp->hw_iwarp_state = iwarp_state;
3163                         nesqp->hw_tcp_state = tcp_state;
3164                         nesqp->last_aeq = async_event_id;
3165                         spin_unlock_irqrestore(&nesqp->lock, flags);
3166                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
3167                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
3168                                         nesqp->hwqp.qp_id);
3169                         if (nesqp->ibqp.event_handler) {
3170                                 ibevent.device = nesqp->ibqp.device;
3171                                 ibevent.element.qp = &nesqp->ibqp;
3172                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3173                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3174                         }
3175                         /* tell cm to disconnect, cm will queue work to thread */
3176                         nes_cm_disconn(nesqp);
3177                         break;
3178                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3179                         nesqp = *((struct nes_qp **)&context);
3180                         spin_lock_irqsave(&nesqp->lock, flags);
3181                         nesqp->hw_iwarp_state = iwarp_state;
3182                         nesqp->hw_tcp_state = tcp_state;
3183                         nesqp->last_aeq = async_event_id;
3184                         spin_unlock_irqrestore(&nesqp->lock, flags);
3185                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
3186                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
3187                                         nesqp->hwqp.qp_id);
3188                         if (nesqp->ibqp.event_handler) {
3189                                 ibevent.device = nesqp->ibqp.device;
3190                                 ibevent.element.qp = &nesqp->ibqp;
3191                                 ibevent.event = IB_EVENT_QP_FATAL;
3192                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3193                         }
3194                         /* tell cm to disconnect, cm will queue work to thread */
3195                         nes_cm_disconn(nesqp);
3196                         break;
3197                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3198                         nesqp = *((struct nes_qp **)&context);
3199                         spin_lock_irqsave(&nesqp->lock, flags);
3200                         nesqp->hw_iwarp_state = iwarp_state;
3201                         nesqp->hw_tcp_state = tcp_state;
3202                         nesqp->last_aeq = async_event_id;
3203                         spin_unlock_irqrestore(&nesqp->lock, flags);
3204                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
3205                                         " event on QP%u \n  Q2 Data:\n",
3206                                         nesqp->hwqp.qp_id);
3207                         if (nesqp->ibqp.event_handler) {
3208                                 ibevent.device = nesqp->ibqp.device;
3209                                 ibevent.element.qp = &nesqp->ibqp;
3210                                 ibevent.event = IB_EVENT_QP_FATAL;
3211                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3212                         }
3213                         /* tell cm to disconnect, cm will queue work to thread */
3214                         nes_cm_disconn(nesqp);
3215                         break;
3216                         /* TODO: additional AEs need to be here */
3217                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3218                         nesqp = *((struct nes_qp **)&context);
3219                         spin_lock_irqsave(&nesqp->lock, flags);
3220                         nesqp->hw_iwarp_state = iwarp_state;
3221                         nesqp->hw_tcp_state = tcp_state;
3222                         nesqp->last_aeq = async_event_id;
3223                         spin_unlock_irqrestore(&nesqp->lock, flags);
3224                         if (nesqp->ibqp.event_handler) {
3225                                 ibevent.device = nesqp->ibqp.device;
3226                                 ibevent.element.qp = &nesqp->ibqp;
3227                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3228                                 nesqp->ibqp.event_handler(&ibevent,
3229                                                 nesqp->ibqp.qp_context);
3230                         }
3231                         nes_cm_disconn(nesqp);
3232                         break;
3233                 default:
3234                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3235                                         async_event_id);
3236                         break;
3237         }
3238
3239 }
3240
3241
3242 /**
3243  * nes_iwarp_ce_handler
3244  */
3245 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3246 {
3247         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3248
3249         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3250                         nescq->hw_cq.cq_number); */
3251         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3252
3253         if (nescq->ibcq.comp_handler)
3254                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3255
3256         return;
3257 }
3258
3259
3260 /**
3261  * nes_manage_apbvt()
3262  */
3263 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3264                 u32 nic_index, u32 add_port)
3265 {
3266         struct nes_device *nesdev = nesvnic->nesdev;
3267         struct nes_hw_cqp_wqe *cqp_wqe;
3268         struct nes_cqp_request *cqp_request;
3269         int ret = 0;
3270         u16 major_code;
3271
3272         /* Send manage APBVT request to CQP */
3273         cqp_request = nes_get_cqp_request(nesdev);
3274         if (cqp_request == NULL) {
3275                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3276                 return -ENOMEM;
3277         }
3278         cqp_request->waiting = 1;
3279         cqp_wqe = &cqp_request->cqp_wqe;
3280
3281         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3282                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3283                         accel_local_port, accel_local_port, nic_index);
3284
3285         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3286         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3287                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3288         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3289                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3290
3291         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3292
3293         atomic_set(&cqp_request->refcount, 2);
3294         nes_post_cqp_request(nesdev, cqp_request);
3295
3296         if (add_port == NES_MANAGE_APBVT_ADD)
3297                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3298                                 NES_EVENT_TIMEOUT);
3299         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3300                         ret, cqp_request->major_code, cqp_request->minor_code);
3301         major_code = cqp_request->major_code;
3302
3303         nes_put_cqp_request(nesdev, cqp_request);
3304
3305         if (!ret)
3306                 return -ETIME;
3307         else if (major_code)
3308                 return -EIO;
3309         else
3310                 return 0;
3311 }
3312
3313
3314 /**
3315  * nes_manage_arp_cache
3316  */
3317 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3318                 u32 ip_addr, u32 action)
3319 {
3320         struct nes_hw_cqp_wqe *cqp_wqe;
3321         struct nes_vnic *nesvnic = netdev_priv(netdev);
3322         struct nes_device *nesdev;
3323         struct nes_cqp_request *cqp_request;
3324         int arp_index;
3325
3326         nesdev = nesvnic->nesdev;
3327         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3328         if (arp_index == -1) {
3329                 return;
3330         }
3331
3332         /* update the ARP entry */
3333         cqp_request = nes_get_cqp_request(nesdev);
3334         if (cqp_request == NULL) {
3335                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3336                 return;
3337         }
3338         cqp_request->waiting = 0;
3339         cqp_wqe = &cqp_request->cqp_wqe;
3340         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3341
3342         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3343                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3344         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3345                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3346         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3347
3348         if (action == NES_ARP_ADD) {
3349                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3350                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3351                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3352                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3353                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3354                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3355         } else {
3356                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3357                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3358         }
3359
3360         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3361                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3362
3363         atomic_set(&cqp_request->refcount, 1);
3364         nes_post_cqp_request(nesdev, cqp_request);
3365 }
3366
3367
3368 /**
3369  * flush_wqes
3370  */
3371 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3372                 u32 which_wq, u32 wait_completion)
3373 {
3374         struct nes_cqp_request *cqp_request;
3375         struct nes_hw_cqp_wqe *cqp_wqe;
3376         int ret;
3377
3378         cqp_request = nes_get_cqp_request(nesdev);
3379         if (cqp_request == NULL) {
3380                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3381                 return;
3382         }
3383         if (wait_completion) {
3384                 cqp_request->waiting = 1;
3385                 atomic_set(&cqp_request->refcount, 2);
3386         } else {
3387                 cqp_request->waiting = 0;
3388         }
3389         cqp_wqe = &cqp_request->cqp_wqe;
3390         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3391
3392         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3393                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3394         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3395
3396         nes_post_cqp_request(nesdev, cqp_request);
3397
3398         if (wait_completion) {
3399                 /* Wait for CQP */
3400                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3401                                 NES_EVENT_TIMEOUT);
3402                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3403                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3404                                 ret, cqp_request->major_code, cqp_request->minor_code);
3405                 nes_put_cqp_request(nesdev, cqp_request);
3406         }
3407 }