drivers/net/ethernet/intel/e1000/e1000_main.c

   1 /*******************************************************************************
   2
   3   Intel PRO/1000 Linux driver
   4   Copyright(c) 1999 - 2006 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   Linux NICS <linux.nics@intel.com>
  24   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  25   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  26
  27 *******************************************************************************/
  28
  29 #include "e1000.h"
  30 #include <net/ip6_checksum.h>
  31 #include <linux/io.h>
  32 #include <linux/prefetch.h>
  33 #include <linux/bitops.h>
  34 #include <linux/if_vlan.h>
  35
  36 char e1000_driver_name[] = "e1000";
  37 static char e1000_driver_string[] = "Intel(R) PRO/1000 Network Driver";
  38 #define DRV_VERSION "7.3.21-k8-NAPI"
  39 const char e1000_driver_version[] = DRV_VERSION;
  40 static const char e1000_copyright[] = "Copyright (c) 1999-2006 Intel Corporation.";
  41
  42 /* e1000_pci_tbl - PCI Device ID Table
  43  *
  44  * Last entry must be all 0s
  45  *
  46  * Macro expands to...
  47  *   {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
  48  */
  49 static const struct pci_device_id e1000_pci_tbl[] = {
  50         INTEL_E1000_ETHERNET_DEVICE(0x1000),
  51         INTEL_E1000_ETHERNET_DEVICE(0x1001),
  52         INTEL_E1000_ETHERNET_DEVICE(0x1004),
  53         INTEL_E1000_ETHERNET_DEVICE(0x1008),
  54         INTEL_E1000_ETHERNET_DEVICE(0x1009),
  55         INTEL_E1000_ETHERNET_DEVICE(0x100C),
  56         INTEL_E1000_ETHERNET_DEVICE(0x100D),
  57         INTEL_E1000_ETHERNET_DEVICE(0x100E),
  58         INTEL_E1000_ETHERNET_DEVICE(0x100F),
  59         INTEL_E1000_ETHERNET_DEVICE(0x1010),
  60         INTEL_E1000_ETHERNET_DEVICE(0x1011),
  61         INTEL_E1000_ETHERNET_DEVICE(0x1012),
  62         INTEL_E1000_ETHERNET_DEVICE(0x1013),
  63         INTEL_E1000_ETHERNET_DEVICE(0x1014),
  64         INTEL_E1000_ETHERNET_DEVICE(0x1015),
  65         INTEL_E1000_ETHERNET_DEVICE(0x1016),
  66         INTEL_E1000_ETHERNET_DEVICE(0x1017),
  67         INTEL_E1000_ETHERNET_DEVICE(0x1018),
  68         INTEL_E1000_ETHERNET_DEVICE(0x1019),
  69         INTEL_E1000_ETHERNET_DEVICE(0x101A),
  70         INTEL_E1000_ETHERNET_DEVICE(0x101D),
  71         INTEL_E1000_ETHERNET_DEVICE(0x101E),
  72         INTEL_E1000_ETHERNET_DEVICE(0x1026),
  73         INTEL_E1000_ETHERNET_DEVICE(0x1027),
  74         INTEL_E1000_ETHERNET_DEVICE(0x1028),
  75         INTEL_E1000_ETHERNET_DEVICE(0x1075),
  76         INTEL_E1000_ETHERNET_DEVICE(0x1076),
  77         INTEL_E1000_ETHERNET_DEVICE(0x1077),
  78         INTEL_E1000_ETHERNET_DEVICE(0x1078),
  79         INTEL_E1000_ETHERNET_DEVICE(0x1079),
  80         INTEL_E1000_ETHERNET_DEVICE(0x107A),
  81         INTEL_E1000_ETHERNET_DEVICE(0x107B),
  82         INTEL_E1000_ETHERNET_DEVICE(0x107C),
  83         INTEL_E1000_ETHERNET_DEVICE(0x108A),
  84         INTEL_E1000_ETHERNET_DEVICE(0x1099),
  85         INTEL_E1000_ETHERNET_DEVICE(0x10B5),
  86         INTEL_E1000_ETHERNET_DEVICE(0x2E6E),
  87         /* required last entry */
  88         {0,}
  89 };
  90
  91 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
  92
  93 int e1000_up(struct e1000_adapter *adapter);
  94 void e1000_down(struct e1000_adapter *adapter);
  95 void e1000_reinit_locked(struct e1000_adapter *adapter);
  96 void e1000_reset(struct e1000_adapter *adapter);
  97 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter);
  98 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter);
  99 void e1000_free_all_tx_resources(struct e1000_adapter *adapter);
 100 void e1000_free_all_rx_resources(struct e1000_adapter *adapter);
 101 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
 102                                     struct e1000_tx_ring *txdr);
 103 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
 104                                     struct e1000_rx_ring *rxdr);
 105 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
 106                                     struct e1000_tx_ring *tx_ring);
 107 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
 108                                     struct e1000_rx_ring *rx_ring);
 109 void e1000_update_stats(struct e1000_adapter *adapter);
 110
 111 static int e1000_init_module(void);
 112 static void e1000_exit_module(void);
 113 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
 114 static void e1000_remove(struct pci_dev *pdev);
 115 static int e1000_alloc_queues(struct e1000_adapter *adapter);
 116 static int e1000_sw_init(struct e1000_adapter *adapter);
 117 int e1000_open(struct net_device *netdev);
 118 int e1000_close(struct net_device *netdev);
 119 static void e1000_configure_tx(struct e1000_adapter *adapter);
 120 static void e1000_configure_rx(struct e1000_adapter *adapter);
 121 static void e1000_setup_rctl(struct e1000_adapter *adapter);
 122 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter);
 123 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter);
 124 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
 125                                 struct e1000_tx_ring *tx_ring);
 126 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
 127                                 struct e1000_rx_ring *rx_ring);
 128 static void e1000_set_rx_mode(struct net_device *netdev);
 129 static void e1000_update_phy_info_task(struct work_struct *work);
 130 static void e1000_watchdog(struct work_struct *work);
 131 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work);
 132 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
 133                                     struct net_device *netdev);
 134 static int e1000_change_mtu(struct net_device *netdev, int new_mtu);
 135 static int e1000_set_mac(struct net_device *netdev, void *p);
 136 static irqreturn_t e1000_intr(int irq, void *data);
 137 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
 138                                struct e1000_tx_ring *tx_ring);
 139 static int e1000_clean(struct napi_struct *napi, int budget);
 140 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
 141                                struct e1000_rx_ring *rx_ring,
 142                                int *work_done, int work_to_do);
 143 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
 144                                      struct e1000_rx_ring *rx_ring,
 145                                      int *work_done, int work_to_do);
 146 static void e1000_alloc_dummy_rx_buffers(struct e1000_adapter *adapter,
 147                                          struct e1000_rx_ring *rx_ring,
 148                                          int cleaned_count)
 149 {
 150 }
 151 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
 152                                    struct e1000_rx_ring *rx_ring,
 153                                    int cleaned_count);
 154 static void e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
 155                                          struct e1000_rx_ring *rx_ring,
 156                                          int cleaned_count);
 157 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd);
 158 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
 159                            int cmd);
 160 static void e1000_enter_82542_rst(struct e1000_adapter *adapter);
 161 static void e1000_leave_82542_rst(struct e1000_adapter *adapter);
 162 static void e1000_tx_timeout(struct net_device *dev);
 163 static void e1000_reset_task(struct work_struct *work);
 164 static void e1000_smartspeed(struct e1000_adapter *adapter);
 165 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
 166                                        struct sk_buff *skb);
 167
 168 static bool e1000_vlan_used(struct e1000_adapter *adapter);
 169 static void e1000_vlan_mode(struct net_device *netdev,
 170                             netdev_features_t features);
 171 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
 172                                      bool filter_on);
 173 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
 174                                  __be16 proto, u16 vid);
 175 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
 176                                   __be16 proto, u16 vid);
 177 static void e1000_restore_vlan(struct e1000_adapter *adapter);
 178
 179 #ifdef CONFIG_PM
 180 static int e1000_suspend(struct pci_dev *pdev, pm_message_t state);
 181 static int e1000_resume(struct pci_dev *pdev);
 182 #endif
 183 static void e1000_shutdown(struct pci_dev *pdev);
 184
 185 #ifdef CONFIG_NET_POLL_CONTROLLER
 186 /* for netdump / net console */
 187 static void e1000_netpoll (struct net_device *netdev);
 188 #endif
 189
 190 #define COPYBREAK_DEFAULT 256
 191 static unsigned int copybreak __read_mostly = COPYBREAK_DEFAULT;
 192 module_param(copybreak, uint, 0644);
 193 MODULE_PARM_DESC(copybreak,
 194         "Maximum size of packet that is copied to a new buffer on receive");
 195
 196 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
 197                                                 pci_channel_state_t state);
 198 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev);
 199 static void e1000_io_resume(struct pci_dev *pdev);
 200
 201 static const struct pci_error_handlers e1000_err_handler = {
 202         .error_detected = e1000_io_error_detected,
 203         .slot_reset = e1000_io_slot_reset,
 204         .resume = e1000_io_resume,
 205 };
 206
 207 static struct pci_driver e1000_driver = {
 208         .name     = e1000_driver_name,
 209         .id_table = e1000_pci_tbl,
 210         .probe    = e1000_probe,
 211         .remove   = e1000_remove,
 212 #ifdef CONFIG_PM
 213         /* Power Management Hooks */
 214         .suspend  = e1000_suspend,
 215         .resume   = e1000_resume,
 216 #endif
 217         .shutdown = e1000_shutdown,
 218         .err_handler = &e1000_err_handler
 219 };
 220
 221 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
 222 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
 223 MODULE_LICENSE("GPL");
 224 MODULE_VERSION(DRV_VERSION);
 225
 226 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
 227 static int debug = -1;
 228 module_param(debug, int, 0);
 229 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 230
 231 /**
 232  * e1000_get_hw_dev - return device
 233  * used by hardware layer to print debugging information
 234  *
 235  **/
 236 struct net_device *e1000_get_hw_dev(struct e1000_hw *hw)
 237 {
 238         struct e1000_adapter *adapter = hw->back;
 239         return adapter->netdev;
 240 }
 241
 242 /**
 243  * e1000_init_module - Driver Registration Routine
 244  *
 245  * e1000_init_module is the first routine called when the driver is
 246  * loaded. All it does is register with the PCI subsystem.
 247  **/
 248 static int __init e1000_init_module(void)
 249 {
 250         int ret;
 251         pr_info("%s - version %s\n", e1000_driver_string, e1000_driver_version);
 252
 253         pr_info("%s\n", e1000_copyright);
 254
 255         ret = pci_register_driver(&e1000_driver);
 256         if (copybreak != COPYBREAK_DEFAULT) {
 257                 if (copybreak == 0)
 258                         pr_info("copybreak disabled\n");
 259                 else
 260                         pr_info("copybreak enabled for "
 261                                    "packets <= %u bytes\n", copybreak);
 262         }
 263         return ret;
 264 }
 265
 266 module_init(e1000_init_module);
 267
 268 /**
 269  * e1000_exit_module - Driver Exit Cleanup Routine
 270  *
 271  * e1000_exit_module is called just before the driver is removed
 272  * from memory.
 273  **/
 274 static void __exit e1000_exit_module(void)
 275 {
 276         pci_unregister_driver(&e1000_driver);
 277 }
 278
 279 module_exit(e1000_exit_module);
 280
 281 static int e1000_request_irq(struct e1000_adapter *adapter)
 282 {
 283         struct net_device *netdev = adapter->netdev;
 284         irq_handler_t handler = e1000_intr;
 285         int irq_flags = IRQF_SHARED;
 286         int err;
 287
 288         err = request_irq(adapter->pdev->irq, handler, irq_flags, netdev->name,
 289                           netdev);
 290         if (err) {
 291                 e_err(probe, "Unable to allocate interrupt Error: %d\n", err);
 292         }
 293
 294         return err;
 295 }
 296
 297 static void e1000_free_irq(struct e1000_adapter *adapter)
 298 {
 299         struct net_device *netdev = adapter->netdev;
 300
 301         free_irq(adapter->pdev->irq, netdev);
 302 }
 303
 304 /**
 305  * e1000_irq_disable - Mask off interrupt generation on the NIC
 306  * @adapter: board private structure
 307  **/
 308 static void e1000_irq_disable(struct e1000_adapter *adapter)
 309 {
 310         struct e1000_hw *hw = &adapter->hw;
 311
 312         ew32(IMC, ~0);
 313         E1000_WRITE_FLUSH();
 314         synchronize_irq(adapter->pdev->irq);
 315 }
 316
 317 /**
 318  * e1000_irq_enable - Enable default interrupt generation settings
 319  * @adapter: board private structure
 320  **/
 321 static void e1000_irq_enable(struct e1000_adapter *adapter)
 322 {
 323         struct e1000_hw *hw = &adapter->hw;
 324
 325         ew32(IMS, IMS_ENABLE_MASK);
 326         E1000_WRITE_FLUSH();
 327 }
 328
 329 static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
 330 {
 331         struct e1000_hw *hw = &adapter->hw;
 332         struct net_device *netdev = adapter->netdev;
 333         u16 vid = hw->mng_cookie.vlan_id;
 334         u16 old_vid = adapter->mng_vlan_id;
 335
 336         if (!e1000_vlan_used(adapter))
 337                 return;
 338
 339         if (!test_bit(vid, adapter->active_vlans)) {
 340                 if (hw->mng_cookie.status &
 341                     E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) {
 342                         e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
 343                         adapter->mng_vlan_id = vid;
 344                 } else {
 345                         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
 346                 }
 347                 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) &&
 348                     (vid != old_vid) &&
 349                     !test_bit(old_vid, adapter->active_vlans))
 350                         e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
 351                                                old_vid);
 352         } else {
 353                 adapter->mng_vlan_id = vid;
 354         }
 355 }
 356
 357 static void e1000_init_manageability(struct e1000_adapter *adapter)
 358 {
 359         struct e1000_hw *hw = &adapter->hw;
 360
 361         if (adapter->en_mng_pt) {
 362                 u32 manc = er32(MANC);
 363
 364                 /* disable hardware interception of ARP */
 365                 manc &= ~(E1000_MANC_ARP_EN);
 366
 367                 ew32(MANC, manc);
 368         }
 369 }
 370
 371 static void e1000_release_manageability(struct e1000_adapter *adapter)
 372 {
 373         struct e1000_hw *hw = &adapter->hw;
 374
 375         if (adapter->en_mng_pt) {
 376                 u32 manc = er32(MANC);
 377
 378                 /* re-enable hardware interception of ARP */
 379                 manc |= E1000_MANC_ARP_EN;
 380
 381                 ew32(MANC, manc);
 382         }
 383 }
 384
 385 /**
 386  * e1000_configure - configure the hardware for RX and TX
 387  * @adapter = private board structure
 388  **/
 389 static void e1000_configure(struct e1000_adapter *adapter)
 390 {
 391         struct net_device *netdev = adapter->netdev;
 392         int i;
 393
 394         e1000_set_rx_mode(netdev);
 395
 396         e1000_restore_vlan(adapter);
 397         e1000_init_manageability(adapter);
 398
 399         e1000_configure_tx(adapter);
 400         e1000_setup_rctl(adapter);
 401         e1000_configure_rx(adapter);
 402         /* call E1000_DESC_UNUSED which always leaves
 403          * at least 1 descriptor unused to make sure
 404          * next_to_use != next_to_clean
 405          */
 406         for (i = 0; i < adapter->num_rx_queues; i++) {
 407                 struct e1000_rx_ring *ring = &adapter->rx_ring[i];
 408                 adapter->alloc_rx_buf(adapter, ring,
 409                                       E1000_DESC_UNUSED(ring));
 410         }
 411 }
 412
 413 int e1000_up(struct e1000_adapter *adapter)
 414 {
 415         struct e1000_hw *hw = &adapter->hw;
 416
 417         /* hardware has been reset, we need to reload some things */
 418         e1000_configure(adapter);
 419
 420         clear_bit(__E1000_DOWN, &adapter->flags);
 421
 422         napi_enable(&adapter->napi);
 423
 424         e1000_irq_enable(adapter);
 425
 426         netif_wake_queue(adapter->netdev);
 427
 428         /* fire a link change interrupt to start the watchdog */
 429         ew32(ICS, E1000_ICS_LSC);
 430         return 0;
 431 }
 432
 433 /**
 434  * e1000_power_up_phy - restore link in case the phy was powered down
 435  * @adapter: address of board private structure
 436  *
 437  * The phy may be powered down to save power and turn off link when the
 438  * driver is unloaded and wake on lan is not enabled (among others)
 439  * *** this routine MUST be followed by a call to e1000_reset ***
 440  **/
 441 void e1000_power_up_phy(struct e1000_adapter *adapter)
 442 {
 443         struct e1000_hw *hw = &adapter->hw;
 444         u16 mii_reg = 0;
 445
 446         /* Just clear the power down bit to wake the phy back up */
 447         if (hw->media_type == e1000_media_type_copper) {
 448                 /* according to the manual, the phy will retain its
 449                  * settings across a power-down/up cycle
 450                  */
 451                 e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
 452                 mii_reg &= ~MII_CR_POWER_DOWN;
 453                 e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
 454         }
 455 }
 456
 457 static void e1000_power_down_phy(struct e1000_adapter *adapter)
 458 {
 459         struct e1000_hw *hw = &adapter->hw;
 460
 461         /* Power down the PHY so no link is implied when interface is down *
 462          * The PHY cannot be powered down if any of the following is true *
 463          * (a) WoL is enabled
 464          * (b) AMT is active
 465          * (c) SoL/IDER session is active
 466          */
 467         if (!adapter->wol && hw->mac_type >= e1000_82540 &&
 468            hw->media_type == e1000_media_type_copper) {
 469                 u16 mii_reg = 0;
 470
 471                 switch (hw->mac_type) {
 472                 case e1000_82540:
 473                 case e1000_82545:
 474                 case e1000_82545_rev_3:
 475                 case e1000_82546:
 476                 case e1000_ce4100:
 477                 case e1000_82546_rev_3:
 478                 case e1000_82541:
 479                 case e1000_82541_rev_2:
 480                 case e1000_82547:
 481                 case e1000_82547_rev_2:
 482                         if (er32(MANC) & E1000_MANC_SMBUS_EN)
 483                                 goto out;
 484                         break;
 485                 default:
 486                         goto out;
 487                 }
 488                 e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
 489                 mii_reg |= MII_CR_POWER_DOWN;
 490                 e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
 491                 msleep(1);
 492         }
 493 out:
 494         return;
 495 }
 496
 497 static void e1000_down_and_stop(struct e1000_adapter *adapter)
 498 {
 499         set_bit(__E1000_DOWN, &adapter->flags);
 500
 501         cancel_delayed_work_sync(&adapter->watchdog_task);
 502
 503         /*
 504          * Since the watchdog task can reschedule other tasks, we should cancel
 505          * it first, otherwise we can run into the situation when a work is
 506          * still running after the adapter has been turned down.
 507          */
 508
 509         cancel_delayed_work_sync(&adapter->phy_info_task);
 510         cancel_delayed_work_sync(&adapter->fifo_stall_task);
 511
 512         /* Only kill reset task if adapter is not resetting */
 513         if (!test_bit(__E1000_RESETTING, &adapter->flags))
 514                 cancel_work_sync(&adapter->reset_task);
 515 }
 516
 517 void e1000_down(struct e1000_adapter *adapter)
 518 {
 519         struct e1000_hw *hw = &adapter->hw;
 520         struct net_device *netdev = adapter->netdev;
 521         u32 rctl, tctl;
 522
 523         /* disable receives in the hardware */
 524         rctl = er32(RCTL);
 525         ew32(RCTL, rctl & ~E1000_RCTL_EN);
 526         /* flush and sleep below */
 527
 528         netif_tx_disable(netdev);
 529
 530         /* disable transmits in the hardware */
 531         tctl = er32(TCTL);
 532         tctl &= ~E1000_TCTL_EN;
 533         ew32(TCTL, tctl);
 534         /* flush both disables and wait for them to finish */
 535         E1000_WRITE_FLUSH();
 536         msleep(10);
 537
 538         /* Set the carrier off after transmits have been disabled in the
 539          * hardware, to avoid race conditions with e1000_watchdog() (which
 540          * may be running concurrently to us, checking for the carrier
 541          * bit to decide whether it should enable transmits again). Such
 542          * a race condition would result into transmission being disabled
 543          * in the hardware until the next IFF_DOWN+IFF_UP cycle.
 544          */
 545         netif_carrier_off(netdev);
 546
 547         napi_disable(&adapter->napi);
 548
 549         e1000_irq_disable(adapter);
 550
 551         /* Setting DOWN must be after irq_disable to prevent
 552          * a screaming interrupt.  Setting DOWN also prevents
 553          * tasks from rescheduling.
 554          */
 555         e1000_down_and_stop(adapter);
 556
 557         adapter->link_speed = 0;
 558         adapter->link_duplex = 0;
 559
 560         e1000_reset(adapter);
 561         e1000_clean_all_tx_rings(adapter);
 562         e1000_clean_all_rx_rings(adapter);
 563 }
 564
 565 void e1000_reinit_locked(struct e1000_adapter *adapter)
 566 {
 567         WARN_ON(in_interrupt());
 568         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
 569                 msleep(1);
 570         e1000_down(adapter);
 571         e1000_up(adapter);
 572         clear_bit(__E1000_RESETTING, &adapter->flags);
 573 }
 574
 575 void e1000_reset(struct e1000_adapter *adapter)
 576 {
 577         struct e1000_hw *hw = &adapter->hw;
 578         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
 579         bool legacy_pba_adjust = false;
 580         u16 hwm;
 581
 582         /* Repartition Pba for greater than 9k mtu
 583          * To take effect CTRL.RST is required.
 584          */
 585
 586         switch (hw->mac_type) {
 587         case e1000_82542_rev2_0:
 588         case e1000_82542_rev2_1:
 589         case e1000_82543:
 590         case e1000_82544:
 591         case e1000_82540:
 592         case e1000_82541:
 593         case e1000_82541_rev_2:
 594                 legacy_pba_adjust = true;
 595                 pba = E1000_PBA_48K;
 596                 break;
 597         case e1000_82545:
 598         case e1000_82545_rev_3:
 599         case e1000_82546:
 600         case e1000_ce4100:
 601         case e1000_82546_rev_3:
 602                 pba = E1000_PBA_48K;
 603                 break;
 604         case e1000_82547:
 605         case e1000_82547_rev_2:
 606                 legacy_pba_adjust = true;
 607                 pba = E1000_PBA_30K;
 608                 break;
 609         case e1000_undefined:
 610         case e1000_num_macs:
 611                 break;
 612         }
 613
 614         if (legacy_pba_adjust) {
 615                 if (hw->max_frame_size > E1000_RXBUFFER_8192)
 616                         pba -= 8; /* allocate more FIFO for Tx */
 617
 618                 if (hw->mac_type == e1000_82547) {
 619                         adapter->tx_fifo_head = 0;
 620                         adapter->tx_head_addr = pba << E1000_TX_HEAD_ADDR_SHIFT;
 621                         adapter->tx_fifo_size =
 622                                 (E1000_PBA_40K - pba) << E1000_PBA_BYTES_SHIFT;
 623                         atomic_set(&adapter->tx_fifo_stall, 0);
 624                 }
 625         } else if (hw->max_frame_size >  ETH_FRAME_LEN + ETH_FCS_LEN) {
 626                 /* adjust PBA for jumbo frames */
 627                 ew32(PBA, pba);
 628
 629                 /* To maintain wire speed transmits, the Tx FIFO should be
 630                  * large enough to accommodate two full transmit packets,
 631                  * rounded up to the next 1KB and expressed in KB.  Likewise,
 632                  * the Rx FIFO should be large enough to accommodate at least
 633                  * one full receive packet and is similarly rounded up and
 634                  * expressed in KB.
 635                  */
 636                 pba = er32(PBA);
 637                 /* upper 16 bits has Tx packet buffer allocation size in KB */
 638                 tx_space = pba >> 16;
 639                 /* lower 16 bits has Rx packet buffer allocation size in KB */
 640                 pba &= 0xffff;
 641                 /* the Tx fifo also stores 16 bytes of information about the Tx
 642                  * but don't include ethernet FCS because hardware appends it
 643                  */
 644                 min_tx_space = (hw->max_frame_size +
 645                                 sizeof(struct e1000_tx_desc) -
 646                                 ETH_FCS_LEN) * 2;
 647                 min_tx_space = ALIGN(min_tx_space, 1024);
 648                 min_tx_space >>= 10;
 649                 /* software strips receive CRC, so leave room for it */
 650                 min_rx_space = hw->max_frame_size;
 651                 min_rx_space = ALIGN(min_rx_space, 1024);
 652                 min_rx_space >>= 10;
 653
 654                 /* If current Tx allocation is less than the min Tx FIFO size,
 655                  * and the min Tx FIFO size is less than the current Rx FIFO
 656                  * allocation, take space away from current Rx allocation
 657                  */
 658                 if (tx_space < min_tx_space &&
 659                     ((min_tx_space - tx_space) < pba)) {
 660                         pba = pba - (min_tx_space - tx_space);
 661
 662                         /* PCI/PCIx hardware has PBA alignment constraints */
 663                         switch (hw->mac_type) {
 664                         case e1000_82545 ... e1000_82546_rev_3:
 665                                 pba &= ~(E1000_PBA_8K - 1);
 666                                 break;
 667                         default:
 668                                 break;
 669                         }
 670
 671                         /* if short on Rx space, Rx wins and must trump Tx
 672                          * adjustment or use Early Receive if available
 673                          */
 674                         if (pba < min_rx_space)
 675                                 pba = min_rx_space;
 676                 }
 677         }
 678
 679         ew32(PBA, pba);
 680
 681         /* flow control settings:
 682          * The high water mark must be low enough to fit one full frame
 683          * (or the size used for early receive) above it in the Rx FIFO.
 684          * Set it to the lower of:
 685          * - 90% of the Rx FIFO size, and
 686          * - the full Rx FIFO size minus the early receive size (for parts
 687          *   with ERT support assuming ERT set to E1000_ERT_2048), or
 688          * - the full Rx FIFO size minus one full frame
 689          */
 690         hwm = min(((pba << 10) * 9 / 10),
 691                   ((pba << 10) - hw->max_frame_size));
 692
 693         hw->fc_high_water = hwm & 0xFFF8;       /* 8-byte granularity */
 694         hw->fc_low_water = hw->fc_high_water - 8;
 695         hw->fc_pause_time = E1000_FC_PAUSE_TIME;
 696         hw->fc_send_xon = 1;
 697         hw->fc = hw->original_fc;
 698
 699         /* Allow time for pending master requests to run */
 700         e1000_reset_hw(hw);
 701         if (hw->mac_type >= e1000_82544)
 702                 ew32(WUC, 0);
 703
 704         if (e1000_init_hw(hw))
 705                 e_dev_err("Hardware Error\n");
 706         e1000_update_mng_vlan(adapter);
 707
 708         /* if (adapter->hwflags & HWFLAGS_PHY_PWR_BIT) { */
 709         if (hw->mac_type >= e1000_82544 &&
 710             hw->autoneg == 1 &&
 711             hw->autoneg_advertised == ADVERTISE_1000_FULL) {
 712                 u32 ctrl = er32(CTRL);
 713                 /* clear phy power management bit if we are in gig only mode,
 714                  * which if enabled will attempt negotiation to 100Mb, which
 715                  * can cause a loss of link at power off or driver unload
 716                  */
 717                 ctrl &= ~E1000_CTRL_SWDPIN3;
 718                 ew32(CTRL, ctrl);
 719         }
 720
 721         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
 722         ew32(VET, ETHERNET_IEEE_VLAN_TYPE);
 723
 724         e1000_reset_adaptive(hw);
 725         e1000_phy_get_info(hw, &adapter->phy_info);
 726
 727         e1000_release_manageability(adapter);
 728 }
 729
 730 /* Dump the eeprom for users having checksum issues */
 731 static void e1000_dump_eeprom(struct e1000_adapter *adapter)
 732 {
 733         struct net_device *netdev = adapter->netdev;
 734         struct ethtool_eeprom eeprom;
 735         const struct ethtool_ops *ops = netdev->ethtool_ops;
 736         u8 *data;
 737         int i;
 738         u16 csum_old, csum_new = 0;
 739
 740         eeprom.len = ops->get_eeprom_len(netdev);
 741         eeprom.offset = 0;
 742
 743         data = kmalloc(eeprom.len, GFP_KERNEL);
 744         if (!data)
 745                 return;
 746
 747         ops->get_eeprom(netdev, &eeprom, data);
 748
 749         csum_old = (data[EEPROM_CHECKSUM_REG * 2]) +
 750                    (data[EEPROM_CHECKSUM_REG * 2 + 1] << 8);
 751         for (i = 0; i < EEPROM_CHECKSUM_REG * 2; i += 2)
 752                 csum_new += data[i] + (data[i + 1] << 8);
 753         csum_new = EEPROM_SUM - csum_new;
 754
 755         pr_err("/*********************/\n");
 756         pr_err("Current EEPROM Checksum : 0x%04x\n", csum_old);
 757         pr_err("Calculated              : 0x%04x\n", csum_new);
 758
 759         pr_err("Offset    Values\n");
 760         pr_err("========  ======\n");
 761         print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, data, 128, 0);
 762
 763         pr_err("Include this output when contacting your support provider.\n");
 764         pr_err("This is not a software error! Something bad happened to\n");
 765         pr_err("your hardware or EEPROM image. Ignoring this problem could\n");
 766         pr_err("result in further problems, possibly loss of data,\n");
 767         pr_err("corruption or system hangs!\n");
 768         pr_err("The MAC Address will be reset to 00:00:00:00:00:00,\n");
 769         pr_err("which is invalid and requires you to set the proper MAC\n");
 770         pr_err("address manually before continuing to enable this network\n");
 771         pr_err("device. Please inspect the EEPROM dump and report the\n");
 772         pr_err("issue to your hardware vendor or Intel Customer Support.\n");
 773         pr_err("/*********************/\n");
 774
 775         kfree(data);
 776 }
 777
 778 /**
 779  * e1000_is_need_ioport - determine if an adapter needs ioport resources or not
 780  * @pdev: PCI device information struct
 781  *
 782  * Return true if an adapter needs ioport resources
 783  **/
 784 static int e1000_is_need_ioport(struct pci_dev *pdev)
 785 {
 786         switch (pdev->device) {
 787         case E1000_DEV_ID_82540EM:
 788         case E1000_DEV_ID_82540EM_LOM:
 789         case E1000_DEV_ID_82540EP:
 790         case E1000_DEV_ID_82540EP_LOM:
 791         case E1000_DEV_ID_82540EP_LP:
 792         case E1000_DEV_ID_82541EI:
 793         case E1000_DEV_ID_82541EI_MOBILE:
 794         case E1000_DEV_ID_82541ER:
 795         case E1000_DEV_ID_82541ER_LOM:
 796         case E1000_DEV_ID_82541GI:
 797         case E1000_DEV_ID_82541GI_LF:
 798         case E1000_DEV_ID_82541GI_MOBILE:
 799         case E1000_DEV_ID_82544EI_COPPER:
 800         case E1000_DEV_ID_82544EI_FIBER:
 801         case E1000_DEV_ID_82544GC_COPPER:
 802         case E1000_DEV_ID_82544GC_LOM:
 803         case E1000_DEV_ID_82545EM_COPPER:
 804         case E1000_DEV_ID_82545EM_FIBER:
 805         case E1000_DEV_ID_82546EB_COPPER:
 806         case E1000_DEV_ID_82546EB_FIBER:
 807         case E1000_DEV_ID_82546EB_QUAD_COPPER:
 808                 return true;
 809         default:
 810                 return false;
 811         }
 812 }
 813
 814 static netdev_features_t e1000_fix_features(struct net_device *netdev,
 815         netdev_features_t features)
 816 {
 817         /* Since there is no support for separate Rx/Tx vlan accel
 818          * enable/disable make sure Tx flag is always in same state as Rx.
 819          */
 820         if (features & NETIF_F_HW_VLAN_CTAG_RX)
 821                 features |= NETIF_F_HW_VLAN_CTAG_TX;
 822         else
 823                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
 824
 825         return features;
 826 }
 827
 828 static int e1000_set_features(struct net_device *netdev,
 829         netdev_features_t features)
 830 {
 831         struct e1000_adapter *adapter = netdev_priv(netdev);
 832         netdev_features_t changed = features ^ netdev->features;
 833
 834         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
 835                 e1000_vlan_mode(netdev, features);
 836
 837         if (!(changed & (NETIF_F_RXCSUM | NETIF_F_RXALL)))
 838                 return 0;
 839
 840         netdev->features = features;
 841         adapter->rx_csum = !!(features & NETIF_F_RXCSUM);
 842
 843         if (netif_running(netdev))
 844                 e1000_reinit_locked(adapter);
 845         else
 846                 e1000_reset(adapter);
 847
 848         return 0;
 849 }
 850
 851 static const struct net_device_ops e1000_netdev_ops = {
 852         .ndo_open               = e1000_open,
 853         .ndo_stop               = e1000_close,
 854         .ndo_start_xmit         = e1000_xmit_frame,
 855         .ndo_set_rx_mode        = e1000_set_rx_mode,
 856         .ndo_set_mac_address    = e1000_set_mac,
 857         .ndo_tx_timeout         = e1000_tx_timeout,
 858         .ndo_change_mtu         = e1000_change_mtu,
 859         .ndo_do_ioctl           = e1000_ioctl,
 860         .ndo_validate_addr      = eth_validate_addr,
 861         .ndo_vlan_rx_add_vid    = e1000_vlan_rx_add_vid,
 862         .ndo_vlan_rx_kill_vid   = e1000_vlan_rx_kill_vid,
 863 #ifdef CONFIG_NET_POLL_CONTROLLER
 864         .ndo_poll_controller    = e1000_netpoll,
 865 #endif
 866         .ndo_fix_features       = e1000_fix_features,
 867         .ndo_set_features       = e1000_set_features,
 868 };
 869
 870 /**
 871  * e1000_init_hw_struct - initialize members of hw struct
 872  * @adapter: board private struct
 873  * @hw: structure used by e1000_hw.c
 874  *
 875  * Factors out initialization of the e1000_hw struct to its own function
 876  * that can be called very early at init (just after struct allocation).
 877  * Fields are initialized based on PCI device information and
 878  * OS network device settings (MTU size).
 879  * Returns negative error codes if MAC type setup fails.
 880  */
 881 static int e1000_init_hw_struct(struct e1000_adapter *adapter,
 882                                 struct e1000_hw *hw)
 883 {
 884         struct pci_dev *pdev = adapter->pdev;
 885
 886         /* PCI config space info */
 887         hw->vendor_id = pdev->vendor;
 888         hw->device_id = pdev->device;
 889         hw->subsystem_vendor_id = pdev->subsystem_vendor;
 890         hw->subsystem_id = pdev->subsystem_device;
 891         hw->revision_id = pdev->revision;
 892
 893         pci_read_config_word(pdev, PCI_COMMAND, &hw->pci_cmd_word);
 894
 895         hw->max_frame_size = adapter->netdev->mtu +
 896                              ENET_HEADER_SIZE + ETHERNET_FCS_SIZE;
 897         hw->min_frame_size = MINIMUM_ETHERNET_FRAME_SIZE;
 898
 899         /* identify the MAC */
 900         if (e1000_set_mac_type(hw)) {
 901                 e_err(probe, "Unknown MAC Type\n");
 902                 return -EIO;
 903         }
 904
 905         switch (hw->mac_type) {
 906         default:
 907                 break;
 908         case e1000_82541:
 909         case e1000_82547:
 910         case e1000_82541_rev_2:
 911         case e1000_82547_rev_2:
 912                 hw->phy_init_script = 1;
 913                 break;
 914         }
 915
 916         e1000_set_media_type(hw);
 917         e1000_get_bus_info(hw);
 918
 919         hw->wait_autoneg_complete = false;
 920         hw->tbi_compatibility_en = true;
 921         hw->adaptive_ifs = true;
 922
 923         /* Copper options */
 924
 925         if (hw->media_type == e1000_media_type_copper) {
 926                 hw->mdix = AUTO_ALL_MODES;
 927                 hw->disable_polarity_correction = false;
 928                 hw->master_slave = E1000_MASTER_SLAVE;
 929         }
 930
 931         return 0;
 932 }
 933
 934 /**
 935  * e1000_probe - Device Initialization Routine
 936  * @pdev: PCI device information struct
 937  * @ent: entry in e1000_pci_tbl
 938  *
 939  * Returns 0 on success, negative on failure
 940  *
 941  * e1000_probe initializes an adapter identified by a pci_dev structure.
 942  * The OS initialization, configuring of the adapter private structure,
 943  * and a hardware reset occur.
 944  **/
 945 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 946 {
 947         struct net_device *netdev;
 948         struct e1000_adapter *adapter = NULL;
 949         struct e1000_hw *hw;
 950
 951         static int cards_found;
 952         static int global_quad_port_a; /* global ksp3 port a indication */
 953         int i, err, pci_using_dac;
 954         u16 eeprom_data = 0;
 955         u16 tmp = 0;
 956         u16 eeprom_apme_mask = E1000_EEPROM_APME;
 957         int bars, need_ioport;
 958         bool disable_dev = false;
 959
 960         /* do not allocate ioport bars when not needed */
 961         need_ioport = e1000_is_need_ioport(pdev);
 962         if (need_ioport) {
 963                 bars = pci_select_bars(pdev, IORESOURCE_MEM | IORESOURCE_IO);
 964                 err = pci_enable_device(pdev);
 965         } else {
 966                 bars = pci_select_bars(pdev, IORESOURCE_MEM);
 967                 err = pci_enable_device_mem(pdev);
 968         }
 969         if (err)
 970                 return err;
 971
 972         err = pci_request_selected_regions(pdev, bars, e1000_driver_name);
 973         if (err)
 974                 goto err_pci_reg;
 975
 976         pci_set_master(pdev);
 977         err = pci_save_state(pdev);
 978         if (err)
 979                 goto err_alloc_etherdev;
 980
 981         err = -ENOMEM;
 982         netdev = alloc_etherdev(sizeof(struct e1000_adapter));
 983         if (!netdev)
 984                 goto err_alloc_etherdev;
 985
 986         SET_NETDEV_DEV(netdev, &pdev->dev);
 987
 988         pci_set_drvdata(pdev, netdev);
 989         adapter = netdev_priv(netdev);
 990         adapter->netdev = netdev;
 991         adapter->pdev = pdev;
 992         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
 993         adapter->bars = bars;
 994         adapter->need_ioport = need_ioport;
 995
 996         hw = &adapter->hw;
 997         hw->back = adapter;
 998
 999         err = -EIO;
1000         hw->hw_addr = pci_ioremap_bar(pdev, BAR_0);
1001         if (!hw->hw_addr)
1002                 goto err_ioremap;
1003
1004         if (adapter->need_ioport) {
1005                 for (i = BAR_1; i <= BAR_5; i++) {
1006                         if (pci_resource_len(pdev, i) == 0)
1007                                 continue;
1008                         if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
1009                                 hw->io_base = pci_resource_start(pdev, i);
1010                                 break;
1011                         }
1012                 }
1013         }
1014
1015         /* make ready for any if (hw->...) below */
1016         err = e1000_init_hw_struct(adapter, hw);
1017         if (err)
1018                 goto err_sw_init;
1019
1020         /* there is a workaround being applied below that limits
1021          * 64-bit DMA addresses to 64-bit hardware.  There are some
1022          * 32-bit adapters that Tx hang when given 64-bit DMA addresses
1023          */
1024         pci_using_dac = 0;
1025         if ((hw->bus_type == e1000_bus_type_pcix) &&
1026             !dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
1027                 pci_using_dac = 1;
1028         } else {
1029                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1030                 if (err) {
1031                         pr_err("No usable DMA config, aborting\n");
1032                         goto err_dma;
1033                 }
1034         }
1035
1036         netdev->netdev_ops = &e1000_netdev_ops;
1037         e1000_set_ethtool_ops(netdev);
1038         netdev->watchdog_timeo = 5 * HZ;
1039         netif_napi_add(netdev, &adapter->napi, e1000_clean, 64);
1040
1041         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1042
1043         adapter->bd_number = cards_found;
1044
1045         /* setup the private structure */
1046
1047         err = e1000_sw_init(adapter);
1048         if (err)
1049                 goto err_sw_init;
1050
1051         err = -EIO;
1052         if (hw->mac_type == e1000_ce4100) {
1053                 hw->ce4100_gbe_mdio_base_virt =
1054                                         ioremap(pci_resource_start(pdev, BAR_1),
1055                                                 pci_resource_len(pdev, BAR_1));
1056
1057                 if (!hw->ce4100_gbe_mdio_base_virt)
1058                         goto err_mdio_ioremap;
1059         }
1060
1061         if (hw->mac_type >= e1000_82543) {
1062                 netdev->hw_features = NETIF_F_SG |
1063                                    NETIF_F_HW_CSUM |
1064                                    NETIF_F_HW_VLAN_CTAG_RX;
1065                 netdev->features = NETIF_F_HW_VLAN_CTAG_TX |
1066                                    NETIF_F_HW_VLAN_CTAG_FILTER;
1067         }
1068
1069         if ((hw->mac_type >= e1000_82544) &&
1070            (hw->mac_type != e1000_82547))
1071                 netdev->hw_features |= NETIF_F_TSO;
1072
1073         netdev->priv_flags |= IFF_SUPP_NOFCS;
1074
1075         netdev->features |= netdev->hw_features;
1076         netdev->hw_features |= (NETIF_F_RXCSUM |
1077                                 NETIF_F_RXALL |
1078                                 NETIF_F_RXFCS);
1079
1080         if (pci_using_dac) {
1081                 netdev->features |= NETIF_F_HIGHDMA;
1082                 netdev->vlan_features |= NETIF_F_HIGHDMA;
1083         }
1084
1085         netdev->vlan_features |= (NETIF_F_TSO |
1086                                   NETIF_F_HW_CSUM |
1087                                   NETIF_F_SG);
1088
1089         /* Do not set IFF_UNICAST_FLT for VMWare's 82545EM */
1090         if (hw->device_id != E1000_DEV_ID_82545EM_COPPER ||
1091             hw->subsystem_vendor_id != PCI_VENDOR_ID_VMWARE)
1092                 netdev->priv_flags |= IFF_UNICAST_FLT;
1093
1094         /* MTU range: 46 - 16110 */
1095         netdev->min_mtu = ETH_ZLEN - ETH_HLEN;
1096         netdev->max_mtu = MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN);
1097
1098         adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
1099
1100         /* initialize eeprom parameters */
1101         if (e1000_init_eeprom_params(hw)) {
1102                 e_err(probe, "EEPROM initialization failed\n");
1103                 goto err_eeprom;
1104         }
1105
1106         /* before reading the EEPROM, reset the controller to
1107          * put the device in a known good starting state
1108          */
1109
1110         e1000_reset_hw(hw);
1111
1112         /* make sure the EEPROM is good */
1113         if (e1000_validate_eeprom_checksum(hw) < 0) {
1114                 e_err(probe, "The EEPROM Checksum Is Not Valid\n");
1115                 e1000_dump_eeprom(adapter);
1116                 /* set MAC address to all zeroes to invalidate and temporary
1117                  * disable this device for the user. This blocks regular
1118                  * traffic while still permitting ethtool ioctls from reaching
1119                  * the hardware as well as allowing the user to run the
1120                  * interface after manually setting a hw addr using
1121                  * `ip set address`
1122                  */
1123                 memset(hw->mac_addr, 0, netdev->addr_len);
1124         } else {
1125                 /* copy the MAC address out of the EEPROM */
1126                 if (e1000_read_mac_addr(hw))
1127                         e_err(probe, "EEPROM Read Error\n");
1128         }
1129         /* don't block initialization here due to bad MAC address */
1130         memcpy(netdev->dev_addr, hw->mac_addr, netdev->addr_len);
1131
1132         if (!is_valid_ether_addr(netdev->dev_addr))
1133                 e_err(probe, "Invalid MAC Address\n");
1134
1135
1136         INIT_DELAYED_WORK(&adapter->watchdog_task, e1000_watchdog);
1137         INIT_DELAYED_WORK(&adapter->fifo_stall_task,
1138                           e1000_82547_tx_fifo_stall_task);
1139         INIT_DELAYED_WORK(&adapter->phy_info_task, e1000_update_phy_info_task);
1140         INIT_WORK(&adapter->reset_task, e1000_reset_task);
1141
1142         e1000_check_options(adapter);
1143
1144         /* Initial Wake on LAN setting
1145          * If APM wake is enabled in the EEPROM,
1146          * enable the ACPI Magic Packet filter
1147          */
1148
1149         switch (hw->mac_type) {
1150         case e1000_82542_rev2_0:
1151         case e1000_82542_rev2_1:
1152         case e1000_82543:
1153                 break;
1154         case e1000_82544:
1155                 e1000_read_eeprom(hw,
1156                         EEPROM_INIT_CONTROL2_REG, 1, &eeprom_data);
1157                 eeprom_apme_mask = E1000_EEPROM_82544_APM;
1158                 break;
1159         case e1000_82546:
1160         case e1000_82546_rev_3:
1161                 if (er32(STATUS) & E1000_STATUS_FUNC_1) {
1162                         e1000_read_eeprom(hw,
1163                                 EEPROM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1164                         break;
1165                 }
1166                 /* Fall Through */
1167         default:
1168                 e1000_read_eeprom(hw,
1169                         EEPROM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1170                 break;
1171         }
1172         if (eeprom_data & eeprom_apme_mask)
1173                 adapter->eeprom_wol |= E1000_WUFC_MAG;
1174
1175         /* now that we have the eeprom settings, apply the special cases
1176          * where the eeprom may be wrong or the board simply won't support
1177          * wake on lan on a particular port
1178          */
1179         switch (pdev->device) {
1180         case E1000_DEV_ID_82546GB_PCIE:
1181                 adapter->eeprom_wol = 0;
1182                 break;
1183         case E1000_DEV_ID_82546EB_FIBER:
1184         case E1000_DEV_ID_82546GB_FIBER:
1185                 /* Wake events only supported on port A for dual fiber
1186                  * regardless of eeprom setting
1187                  */
1188                 if (er32(STATUS) & E1000_STATUS_FUNC_1)
1189                         adapter->eeprom_wol = 0;
1190                 break;
1191         case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
1192                 /* if quad port adapter, disable WoL on all but port A */
1193                 if (global_quad_port_a != 0)
1194                         adapter->eeprom_wol = 0;
1195                 else
1196                         adapter->quad_port_a = true;
1197                 /* Reset for multiple quad port adapters */
1198                 if (++global_quad_port_a == 4)
1199                         global_quad_port_a = 0;
1200                 break;
1201         }
1202
1203         /* initialize the wol settings based on the eeprom settings */
1204         adapter->wol = adapter->eeprom_wol;
1205         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1206
1207         /* Auto detect PHY address */
1208         if (hw->mac_type == e1000_ce4100) {
1209                 for (i = 0; i < 32; i++) {
1210                         hw->phy_addr = i;
1211                         e1000_read_phy_reg(hw, PHY_ID2, &tmp);
1212
1213                         if (tmp != 0 && tmp != 0xFF)
1214                                 break;
1215                 }
1216
1217                 if (i >= 32)
1218                         goto err_eeprom;
1219         }
1220
1221         /* reset the hardware with the new settings */
1222         e1000_reset(adapter);
1223
1224         strcpy(netdev->name, "eth%d");
1225         err = register_netdev(netdev);
1226         if (err)
1227                 goto err_register;
1228
1229         e1000_vlan_filter_on_off(adapter, false);
1230
1231         /* print bus type/speed/width info */
1232         e_info(probe, "(PCI%s:%dMHz:%d-bit) %pM\n",
1233                ((hw->bus_type == e1000_bus_type_pcix) ? "-X" : ""),
1234                ((hw->bus_speed == e1000_bus_speed_133) ? 133 :
1235                 (hw->bus_speed == e1000_bus_speed_120) ? 120 :
1236                 (hw->bus_speed == e1000_bus_speed_100) ? 100 :
1237                 (hw->bus_speed == e1000_bus_speed_66) ? 66 : 33),
1238                ((hw->bus_width == e1000_bus_width_64) ? 64 : 32),
1239                netdev->dev_addr);
1240
1241         /* carrier off reporting is important to ethtool even BEFORE open */
1242         netif_carrier_off(netdev);
1243
1244         e_info(probe, "Intel(R) PRO/1000 Network Connection\n");
1245
1246         cards_found++;
1247         return 0;
1248
1249 err_register:
1250 err_eeprom:
1251         e1000_phy_hw_reset(hw);
1252
1253         if (hw->flash_address)
1254                 iounmap(hw->flash_address);
1255         kfree(adapter->tx_ring);
1256         kfree(adapter->rx_ring);
1257 err_dma:
1258 err_sw_init:
1259 err_mdio_ioremap:
1260         iounmap(hw->ce4100_gbe_mdio_base_virt);
1261         iounmap(hw->hw_addr);
1262 err_ioremap:
1263         disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags);
1264         free_netdev(netdev);
1265 err_alloc_etherdev:
1266         pci_release_selected_regions(pdev, bars);
1267 err_pci_reg:
1268         if (!adapter || disable_dev)
1269                 pci_disable_device(pdev);
1270         return err;
1271 }
1272
1273 /**
1274  * e1000_remove - Device Removal Routine
1275  * @pdev: PCI device information struct
1276  *
1277  * e1000_remove is called by the PCI subsystem to alert the driver
1278  * that it should release a PCI device. That could be caused by a
1279  * Hot-Plug event, or because the driver is going to be removed from
1280  * memory.
1281  **/
1282 static void e1000_remove(struct pci_dev *pdev)
1283 {
1284         struct net_device *netdev = pci_get_drvdata(pdev);
1285         struct e1000_adapter *adapter = netdev_priv(netdev);
1286         struct e1000_hw *hw = &adapter->hw;
1287         bool disable_dev;
1288
1289         e1000_down_and_stop(adapter);
1290         e1000_release_manageability(adapter);
1291
1292         unregister_netdev(netdev);
1293
1294         e1000_phy_hw_reset(hw);
1295
1296         kfree(adapter->tx_ring);
1297         kfree(adapter->rx_ring);
1298
1299         if (hw->mac_type == e1000_ce4100)
1300                 iounmap(hw->ce4100_gbe_mdio_base_virt);
1301         iounmap(hw->hw_addr);
1302         if (hw->flash_address)
1303                 iounmap(hw->flash_address);
1304         pci_release_selected_regions(pdev, adapter->bars);
1305
1306         disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags);
1307         free_netdev(netdev);
1308
1309         if (disable_dev)
1310                 pci_disable_device(pdev);
1311 }
1312
1313 /**
1314  * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
1315  * @adapter: board private structure to initialize
1316  *
1317  * e1000_sw_init initializes the Adapter private data structure.
1318  * e1000_init_hw_struct MUST be called before this function
1319  **/
1320 static int e1000_sw_init(struct e1000_adapter *adapter)
1321 {
1322         adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1323
1324         adapter->num_tx_queues = 1;
1325         adapter->num_rx_queues = 1;
1326
1327         if (e1000_alloc_queues(adapter)) {
1328                 e_err(probe, "Unable to allocate memory for queues\n");
1329                 return -ENOMEM;
1330         }
1331
1332         /* Explicitly disable IRQ since the NIC can be in any state. */
1333         e1000_irq_disable(adapter);
1334
1335         spin_lock_init(&adapter->stats_lock);
1336
1337         set_bit(__E1000_DOWN, &adapter->flags);
1338
1339         return 0;
1340 }
1341
1342 /**
1343  * e1000_alloc_queues - Allocate memory for all rings
1344  * @adapter: board private structure to initialize
1345  *
1346  * We allocate one ring per queue at run-time since we don't know the
1347  * number of queues at compile-time.
1348  **/
1349 static int e1000_alloc_queues(struct e1000_adapter *adapter)
1350 {
1351         adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1352                                    sizeof(struct e1000_tx_ring), GFP_KERNEL);
1353         if (!adapter->tx_ring)
1354                 return -ENOMEM;
1355
1356         adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1357                                    sizeof(struct e1000_rx_ring), GFP_KERNEL);
1358         if (!adapter->rx_ring) {
1359                 kfree(adapter->tx_ring);
1360                 return -ENOMEM;
1361         }
1362
1363         return E1000_SUCCESS;
1364 }
1365
1366 /**
1367  * e1000_open - Called when a network interface is made active
1368  * @netdev: network interface device structure
1369  *
1370  * Returns 0 on success, negative value on failure
1371  *
1372  * The open entry point is called when a network interface is made
1373  * active by the system (IFF_UP).  At this point all resources needed
1374  * for transmit and receive operations are allocated, the interrupt
1375  * handler is registered with the OS, the watchdog task is started,
1376  * and the stack is notified that the interface is ready.
1377  **/
1378 int e1000_open(struct net_device *netdev)
1379 {
1380         struct e1000_adapter *adapter = netdev_priv(netdev);
1381         struct e1000_hw *hw = &adapter->hw;
1382         int err;
1383
1384         /* disallow open during test */
1385         if (test_bit(__E1000_TESTING, &adapter->flags))
1386                 return -EBUSY;
1387
1388         netif_carrier_off(netdev);
1389
1390         /* allocate transmit descriptors */
1391         err = e1000_setup_all_tx_resources(adapter);
1392         if (err)
1393                 goto err_setup_tx;
1394
1395         /* allocate receive descriptors */
1396         err = e1000_setup_all_rx_resources(adapter);
1397         if (err)
1398                 goto err_setup_rx;
1399
1400         e1000_power_up_phy(adapter);
1401
1402         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
1403         if ((hw->mng_cookie.status &
1404                           E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT)) {
1405                 e1000_update_mng_vlan(adapter);
1406         }
1407
1408         /* before we allocate an interrupt, we must be ready to handle it.
1409          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1410          * as soon as we call pci_request_irq, so we have to setup our
1411          * clean_rx handler before we do so.
1412          */
1413         e1000_configure(adapter);
1414
1415         err = e1000_request_irq(adapter);
1416         if (err)
1417                 goto err_req_irq;
1418
1419         /* From here on the code is the same as e1000_up() */
1420         clear_bit(__E1000_DOWN, &adapter->flags);
1421
1422         napi_enable(&adapter->napi);
1423
1424         e1000_irq_enable(adapter);
1425
1426         netif_start_queue(netdev);
1427
1428         /* fire a link status change interrupt to start the watchdog */
1429         ew32(ICS, E1000_ICS_LSC);
1430
1431         return E1000_SUCCESS;
1432
1433 err_req_irq:
1434         e1000_power_down_phy(adapter);
1435         e1000_free_all_rx_resources(adapter);
1436 err_setup_rx:
1437         e1000_free_all_tx_resources(adapter);
1438 err_setup_tx:
1439         e1000_reset(adapter);
1440
1441         return err;
1442 }
1443
1444 /**
1445  * e1000_close - Disables a network interface
1446  * @netdev: network interface device structure
1447  *
1448  * Returns 0, this is not allowed to fail
1449  *
1450  * The close entry point is called when an interface is de-activated
1451  * by the OS.  The hardware is still under the drivers control, but
1452  * needs to be disabled.  A global MAC reset is issued to stop the
1453  * hardware, and all transmit and receive resources are freed.
1454  **/
1455 int e1000_close(struct net_device *netdev)
1456 {
1457         struct e1000_adapter *adapter = netdev_priv(netdev);
1458         struct e1000_hw *hw = &adapter->hw;
1459         int count = E1000_CHECK_RESET_COUNT;
1460
1461         while (test_bit(__E1000_RESETTING, &adapter->flags) && count--)
1462                 usleep_range(10000, 20000);
1463
1464         WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
1465         e1000_down(adapter);
1466         e1000_power_down_phy(adapter);
1467         e1000_free_irq(adapter);
1468
1469         e1000_free_all_tx_resources(adapter);
1470         e1000_free_all_rx_resources(adapter);
1471
1472         /* kill manageability vlan ID if supported, but not if a vlan with
1473          * the same ID is registered on the host OS (let 8021q kill it)
1474          */
1475         if ((hw->mng_cookie.status &
1476              E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
1477             !test_bit(adapter->mng_vlan_id, adapter->active_vlans)) {
1478                 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
1479                                        adapter->mng_vlan_id);
1480         }
1481
1482         return 0;
1483 }
1484
1485 /**
1486  * e1000_check_64k_bound - check that memory doesn't cross 64kB boundary
1487  * @adapter: address of board private structure
1488  * @start: address of beginning of memory
1489  * @len: length of memory
1490  **/
1491 static bool e1000_check_64k_bound(struct e1000_adapter *adapter, void *start,
1492                                   unsigned long len)
1493 {
1494         struct e1000_hw *hw = &adapter->hw;
1495         unsigned long begin = (unsigned long)start;
1496         unsigned long end = begin + len;
1497
1498         /* First rev 82545 and 82546 need to not allow any memory
1499          * write location to cross 64k boundary due to errata 23
1500          */
1501         if (hw->mac_type == e1000_82545 ||
1502             hw->mac_type == e1000_ce4100 ||
1503             hw->mac_type == e1000_82546) {
1504                 return ((begin ^ (end - 1)) >> 16) != 0 ? false : true;
1505         }
1506
1507         return true;
1508 }
1509
1510 /**
1511  * e1000_setup_tx_resources - allocate Tx resources (Descriptors)
1512  * @adapter: board private structure
1513  * @txdr:    tx descriptor ring (for a specific queue) to setup
1514  *
1515  * Return 0 on success, negative on failure
1516  **/
1517 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
1518                                     struct e1000_tx_ring *txdr)
1519 {
1520         struct pci_dev *pdev = adapter->pdev;
1521         int size;
1522
1523         size = sizeof(struct e1000_tx_buffer) * txdr->count;
1524         txdr->buffer_info = vzalloc(size);
1525         if (!txdr->buffer_info)
1526                 return -ENOMEM;
1527
1528         /* round up to nearest 4K */
1529
1530         txdr->size = txdr->count * sizeof(struct e1000_tx_desc);
1531         txdr->size = ALIGN(txdr->size, 4096);
1532
1533         txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size, &txdr->dma,
1534                                         GFP_KERNEL);
1535         if (!txdr->desc) {
1536 setup_tx_desc_die:
1537                 vfree(txdr->buffer_info);
1538                 return -ENOMEM;
1539         }
1540
1541         /* Fix for errata 23, can't cross 64kB boundary */
1542         if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1543                 void *olddesc = txdr->desc;
1544                 dma_addr_t olddma = txdr->dma;
1545                 e_err(tx_err, "txdr align check failed: %u bytes at %p\n",
1546                       txdr->size, txdr->desc);
1547                 /* Try again, without freeing the previous */
1548                 txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size,
1549                                                 &txdr->dma, GFP_KERNEL);
1550                 /* Failed allocation, critical failure */
1551                 if (!txdr->desc) {
1552                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1553                                           olddma);
1554                         goto setup_tx_desc_die;
1555                 }
1556
1557                 if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1558                         /* give up */
1559                         dma_free_coherent(&pdev->dev, txdr->size, txdr->desc,
1560                                           txdr->dma);
1561                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1562                                           olddma);
1563                         e_err(probe, "Unable to allocate aligned memory "
1564                               "for the transmit descriptor ring\n");
1565                         vfree(txdr->buffer_info);
1566                         return -ENOMEM;
1567                 } else {
1568                         /* Free old allocation, new allocation was successful */
1569                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1570                                           olddma);
1571                 }
1572         }
1573         memset(txdr->desc, 0, txdr->size);
1574
1575         txdr->next_to_use = 0;
1576         txdr->next_to_clean = 0;
1577
1578         return 0;
1579 }
1580
1581 /**
1582  * e1000_setup_all_tx_resources - wrapper to allocate Tx resources
1583  *                                (Descriptors) for all queues
1584  * @adapter: board private structure
1585  *
1586  * Return 0 on success, negative on failure
1587  **/
1588 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter)
1589 {
1590         int i, err = 0;
1591
1592         for (i = 0; i < adapter->num_tx_queues; i++) {
1593                 err = e1000_setup_tx_resources(adapter, &adapter->tx_ring[i]);
1594                 if (err) {
1595                         e_err(probe, "Allocation for Tx Queue %u failed\n", i);
1596                         for (i-- ; i >= 0; i--)
1597                                 e1000_free_tx_resources(adapter,
1598                                                         &adapter->tx_ring[i]);
1599                         break;
1600                 }
1601         }
1602
1603         return err;
1604 }
1605
1606 /**
1607  * e1000_configure_tx - Configure 8254x Transmit Unit after Reset
1608  * @adapter: board private structure
1609  *
1610  * Configure the Tx unit of the MAC after a reset.
1611  **/
1612 static void e1000_configure_tx(struct e1000_adapter *adapter)
1613 {
1614         u64 tdba;
1615         struct e1000_hw *hw = &adapter->hw;
1616         u32 tdlen, tctl, tipg;
1617         u32 ipgr1, ipgr2;
1618
1619         /* Setup the HW Tx Head and Tail descriptor pointers */
1620
1621         switch (adapter->num_tx_queues) {
1622         case 1:
1623         default:
1624                 tdba = adapter->tx_ring[0].dma;
1625                 tdlen = adapter->tx_ring[0].count *
1626                         sizeof(struct e1000_tx_desc);
1627                 ew32(TDLEN, tdlen);
1628                 ew32(TDBAH, (tdba >> 32));
1629                 ew32(TDBAL, (tdba & 0x00000000ffffffffULL));
1630                 ew32(TDT, 0);
1631                 ew32(TDH, 0);
1632                 adapter->tx_ring[0].tdh = ((hw->mac_type >= e1000_82543) ?
1633                                            E1000_TDH : E1000_82542_TDH);
1634                 adapter->tx_ring[0].tdt = ((hw->mac_type >= e1000_82543) ?
1635                                            E1000_TDT : E1000_82542_TDT);
1636                 break;
1637         }
1638
1639         /* Set the default values for the Tx Inter Packet Gap timer */
1640         if ((hw->media_type == e1000_media_type_fiber ||
1641              hw->media_type == e1000_media_type_internal_serdes))
1642                 tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1643         else
1644                 tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1645
1646         switch (hw->mac_type) {
1647         case e1000_82542_rev2_0:
1648         case e1000_82542_rev2_1:
1649                 tipg = DEFAULT_82542_TIPG_IPGT;
1650                 ipgr1 = DEFAULT_82542_TIPG_IPGR1;
1651                 ipgr2 = DEFAULT_82542_TIPG_IPGR2;
1652                 break;
1653         default:
1654                 ipgr1 = DEFAULT_82543_TIPG_IPGR1;
1655                 ipgr2 = DEFAULT_82543_TIPG_IPGR2;
1656                 break;
1657         }
1658         tipg |= ipgr1 << E1000_TIPG_IPGR1_SHIFT;
1659         tipg |= ipgr2 << E1000_TIPG_IPGR2_SHIFT;
1660         ew32(TIPG, tipg);
1661
1662         /* Set the Tx Interrupt Delay register */
1663
1664         ew32(TIDV, adapter->tx_int_delay);
1665         if (hw->mac_type >= e1000_82540)
1666                 ew32(TADV, adapter->tx_abs_int_delay);
1667
1668         /* Program the Transmit Control Register */
1669
1670         tctl = er32(TCTL);
1671         tctl &= ~E1000_TCTL_CT;
1672         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
1673                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1674
1675         e1000_config_collision_dist(hw);
1676
1677         /* Setup Transmit Descriptor Settings for eop descriptor */
1678         adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
1679
1680         /* only set IDE if we are delaying interrupts using the timers */
1681         if (adapter->tx_int_delay)
1682                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
1683
1684         if (hw->mac_type < e1000_82543)
1685                 adapter->txd_cmd |= E1000_TXD_CMD_RPS;
1686         else
1687                 adapter->txd_cmd |= E1000_TXD_CMD_RS;
1688
1689         /* Cache if we're 82544 running in PCI-X because we'll
1690          * need this to apply a workaround later in the send path.
1691          */
1692         if (hw->mac_type == e1000_82544 &&
1693             hw->bus_type == e1000_bus_type_pcix)
1694                 adapter->pcix_82544 = true;
1695
1696         ew32(TCTL, tctl);
1697
1698 }
1699
1700 /**
1701  * e1000_setup_rx_resources - allocate Rx resources (Descriptors)
1702  * @adapter: board private structure
1703  * @rxdr:    rx descriptor ring (for a specific queue) to setup
1704  *
1705  * Returns 0 on success, negative on failure
1706  **/
1707 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
1708                                     struct e1000_rx_ring *rxdr)
1709 {
1710         struct pci_dev *pdev = adapter->pdev;
1711         int size, desc_len;
1712
1713         size = sizeof(struct e1000_rx_buffer) * rxdr->count;
1714         rxdr->buffer_info = vzalloc(size);
1715         if (!rxdr->buffer_info)
1716                 return -ENOMEM;
1717
1718         desc_len = sizeof(struct e1000_rx_desc);
1719
1720         /* Round up to nearest 4K */
1721
1722         rxdr->size = rxdr->count * desc_len;
1723         rxdr->size = ALIGN(rxdr->size, 4096);
1724
1725         rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size, &rxdr->dma,
1726                                         GFP_KERNEL);
1727         if (!rxdr->desc) {
1728 setup_rx_desc_die:
1729                 vfree(rxdr->buffer_info);
1730                 return -ENOMEM;
1731         }
1732
1733         /* Fix for errata 23, can't cross 64kB boundary */
1734         if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1735                 void *olddesc = rxdr->desc;
1736                 dma_addr_t olddma = rxdr->dma;
1737                 e_err(rx_err, "rxdr align check failed: %u bytes at %p\n",
1738                       rxdr->size, rxdr->desc);
1739                 /* Try again, without freeing the previous */
1740                 rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size,
1741                                                 &rxdr->dma, GFP_KERNEL);
1742                 /* Failed allocation, critical failure */
1743                 if (!rxdr->desc) {
1744                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1745                                           olddma);
1746                         goto setup_rx_desc_die;
1747                 }
1748
1749                 if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1750                         /* give up */
1751                         dma_free_coherent(&pdev->dev, rxdr->size, rxdr->desc,
1752                                           rxdr->dma);
1753                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1754                                           olddma);
1755                         e_err(probe, "Unable to allocate aligned memory for "
1756                               "the Rx descriptor ring\n");
1757                         goto setup_rx_desc_die;
1758                 } else {
1759                         /* Free old allocation, new allocation was successful */
1760                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1761                                           olddma);
1762                 }
1763         }
1764         memset(rxdr->desc, 0, rxdr->size);
1765
1766         rxdr->next_to_clean = 0;
1767         rxdr->next_to_use = 0;
1768         rxdr->rx_skb_top = NULL;
1769
1770         return 0;
1771 }
1772
1773 /**
1774  * e1000_setup_all_rx_resources - wrapper to allocate Rx resources
1775  *                                (Descriptors) for all queues
1776  * @adapter: board private structure
1777  *
1778  * Return 0 on success, negative on failure
1779  **/
1780 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter)
1781 {
1782         int i, err = 0;
1783
1784         for (i = 0; i < adapter->num_rx_queues; i++) {
1785                 err = e1000_setup_rx_resources(adapter, &adapter->rx_ring[i]);
1786                 if (err) {
1787                         e_err(probe, "Allocation for Rx Queue %u failed\n", i);
1788                         for (i-- ; i >= 0; i--)
1789                                 e1000_free_rx_resources(adapter,
1790                                                         &adapter->rx_ring[i]);
1791                         break;
1792                 }
1793         }
1794
1795         return err;
1796 }
1797
1798 /**
1799  * e1000_setup_rctl - configure the receive control registers
1800  * @adapter: Board private structure
1801  **/
1802 static void e1000_setup_rctl(struct e1000_adapter *adapter)
1803 {
1804         struct e1000_hw *hw = &adapter->hw;
1805         u32 rctl;
1806
1807         rctl = er32(RCTL);
1808
1809         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
1810
1811         rctl |= E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
1812                 E1000_RCTL_RDMTS_HALF |
1813                 (hw->mc_filter_type << E1000_RCTL_MO_SHIFT);
1814
1815         if (hw->tbi_compatibility_on == 1)
1816                 rctl |= E1000_RCTL_SBP;
1817         else
1818                 rctl &= ~E1000_RCTL_SBP;
1819
1820         if (adapter->netdev->mtu <= ETH_DATA_LEN)
1821                 rctl &= ~E1000_RCTL_LPE;
1822         else
1823                 rctl |= E1000_RCTL_LPE;
1824
1825         /* Setup buffer sizes */
1826         rctl &= ~E1000_RCTL_SZ_4096;
1827         rctl |= E1000_RCTL_BSEX;
1828         switch (adapter->rx_buffer_len) {
1829         case E1000_RXBUFFER_2048:
1830         default:
1831                 rctl |= E1000_RCTL_SZ_2048;
1832                 rctl &= ~E1000_RCTL_BSEX;
1833                 break;
1834         case E1000_RXBUFFER_4096:
1835                 rctl |= E1000_RCTL_SZ_4096;
1836                 break;
1837         case E1000_RXBUFFER_8192:
1838                 rctl |= E1000_RCTL_SZ_8192;
1839                 break;
1840         case E1000_RXBUFFER_16384:
1841                 rctl |= E1000_RCTL_SZ_16384;
1842                 break;
1843         }
1844
1845         /* This is useful for sniffing bad packets. */
1846         if (adapter->netdev->features & NETIF_F_RXALL) {
1847                 /* UPE and MPE will be handled by normal PROMISC logic
1848                  * in e1000e_set_rx_mode
1849                  */
1850                 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
1851                          E1000_RCTL_BAM | /* RX All Bcast Pkts */
1852                          E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
1853
1854                 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
1855                           E1000_RCTL_DPF | /* Allow filtered pause */
1856                           E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
1857                 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
1858                  * and that breaks VLANs.
1859                  */
1860         }
1861
1862         ew32(RCTL, rctl);
1863 }
1864
1865 /**
1866  * e1000_configure_rx - Configure 8254x Receive Unit after Reset
1867  * @adapter: board private structure
1868  *
1869  * Configure the Rx unit of the MAC after a reset.
1870  **/
1871 static void e1000_configure_rx(struct e1000_adapter *adapter)
1872 {
1873         u64 rdba;
1874         struct e1000_hw *hw = &adapter->hw;
1875         u32 rdlen, rctl, rxcsum;
1876
1877         if (adapter->netdev->mtu > ETH_DATA_LEN) {
1878                 rdlen = adapter->rx_ring[0].count *
1879                         sizeof(struct e1000_rx_desc);
1880                 adapter->clean_rx = e1000_clean_jumbo_rx_irq;
1881                 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
1882         } else {
1883                 rdlen = adapter->rx_ring[0].count *
1884                         sizeof(struct e1000_rx_desc);
1885                 adapter->clean_rx = e1000_clean_rx_irq;
1886                 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
1887         }
1888
1889         /* disable receives while setting up the descriptors */
1890         rctl = er32(RCTL);
1891         ew32(RCTL, rctl & ~E1000_RCTL_EN);
1892
1893         /* set the Receive Delay Timer Register */
1894         ew32(RDTR, adapter->rx_int_delay);
1895
1896         if (hw->mac_type >= e1000_82540) {
1897                 ew32(RADV, adapter->rx_abs_int_delay);
1898                 if (adapter->itr_setting != 0)
1899                         ew32(ITR, 1000000000 / (adapter->itr * 256));
1900         }
1901
1902         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1903          * the Base and Length of the Rx Descriptor Ring
1904          */
1905         switch (adapter->num_rx_queues) {
1906         case 1:
1907         default:
1908                 rdba = adapter->rx_ring[0].dma;
1909                 ew32(RDLEN, rdlen);
1910                 ew32(RDBAH, (rdba >> 32));
1911                 ew32(RDBAL, (rdba & 0x00000000ffffffffULL));
1912                 ew32(RDT, 0);
1913                 ew32(RDH, 0);
1914                 adapter->rx_ring[0].rdh = ((hw->mac_type >= e1000_82543) ?
1915                                            E1000_RDH : E1000_82542_RDH);
1916                 adapter->rx_ring[0].rdt = ((hw->mac_type >= e1000_82543) ?
1917                                            E1000_RDT : E1000_82542_RDT);
1918                 break;
1919         }
1920
1921         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
1922         if (hw->mac_type >= e1000_82543) {
1923                 rxcsum = er32(RXCSUM);
1924                 if (adapter->rx_csum)
1925                         rxcsum |= E1000_RXCSUM_TUOFL;
1926                 else
1927                         /* don't need to clear IPPCSE as it defaults to 0 */
1928                         rxcsum &= ~E1000_RXCSUM_TUOFL;
1929                 ew32(RXCSUM, rxcsum);
1930         }
1931
1932         /* Enable Receives */
1933         ew32(RCTL, rctl | E1000_RCTL_EN);
1934 }
1935
1936 /**
1937  * e1000_free_tx_resources - Free Tx Resources per Queue
1938  * @adapter: board private structure
1939  * @tx_ring: Tx descriptor ring for a specific queue
1940  *
1941  * Free all transmit software resources
1942  **/
1943 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
1944                                     struct e1000_tx_ring *tx_ring)
1945 {
1946         struct pci_dev *pdev = adapter->pdev;
1947
1948         e1000_clean_tx_ring(adapter, tx_ring);
1949
1950         vfree(tx_ring->buffer_info);
1951         tx_ring->buffer_info = NULL;
1952
1953         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
1954                           tx_ring->dma);
1955
1956         tx_ring->desc = NULL;
1957 }
1958
1959 /**
1960  * e1000_free_all_tx_resources - Free Tx Resources for All Queues
1961  * @adapter: board private structure
1962  *
1963  * Free all transmit software resources
1964  **/
1965 void e1000_free_all_tx_resources(struct e1000_adapter *adapter)
1966 {
1967         int i;
1968
1969         for (i = 0; i < adapter->num_tx_queues; i++)
1970                 e1000_free_tx_resources(adapter, &adapter->tx_ring[i]);
1971 }
1972
1973 static void
1974 e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter,
1975                                  struct e1000_tx_buffer *buffer_info)
1976 {
1977         if (buffer_info->dma) {
1978                 if (buffer_info->mapped_as_page)
1979                         dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1980                                        buffer_info->length, DMA_TO_DEVICE);
1981                 else
1982                         dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1983                                          buffer_info->length,
1984                                          DMA_TO_DEVICE);
1985                 buffer_info->dma = 0;
1986         }
1987         if (buffer_info->skb) {
1988                 dev_kfree_skb_any(buffer_info->skb);
1989                 buffer_info->skb = NULL;
1990         }
1991         buffer_info->time_stamp = 0;
1992         /* buffer_info must be completely set up in the transmit path */
1993 }
1994
1995 /**
1996  * e1000_clean_tx_ring - Free Tx Buffers
1997  * @adapter: board private structure
1998  * @tx_ring: ring to be cleaned
1999  **/
2000 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
2001                                 struct e1000_tx_ring *tx_ring)
2002 {
2003         struct e1000_hw *hw = &adapter->hw;
2004         struct e1000_tx_buffer *buffer_info;
2005         unsigned long size;
2006         unsigned int i;
2007
2008         /* Free all the Tx ring sk_buffs */
2009
2010         for (i = 0; i < tx_ring->count; i++) {
2011                 buffer_info = &tx_ring->buffer_info[i];
2012                 e1000_unmap_and_free_tx_resource(adapter, buffer_info);
2013         }
2014
2015         netdev_reset_queue(adapter->netdev);
2016         size = sizeof(struct e1000_tx_buffer) * tx_ring->count;
2017         memset(tx_ring->buffer_info, 0, size);
2018
2019         /* Zero out the descriptor ring */
2020
2021         memset(tx_ring->desc, 0, tx_ring->size);
2022
2023         tx_ring->next_to_use = 0;
2024         tx_ring->next_to_clean = 0;
2025         tx_ring->last_tx_tso = false;
2026
2027         writel(0, hw->hw_addr + tx_ring->tdh);
2028         writel(0, hw->hw_addr + tx_ring->tdt);
2029 }
2030
2031 /**
2032  * e1000_clean_all_tx_rings - Free Tx Buffers for all queues
2033  * @adapter: board private structure
2034  **/
2035 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter)
2036 {
2037         int i;
2038
2039         for (i = 0; i < adapter->num_tx_queues; i++)
2040                 e1000_clean_tx_ring(adapter, &adapter->tx_ring[i]);
2041 }
2042
2043 /**
2044  * e1000_free_rx_resources - Free Rx Resources
2045  * @adapter: board private structure
2046  * @rx_ring: ring to clean the resources from
2047  *
2048  * Free all receive software resources
2049  **/
2050 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
2051                                     struct e1000_rx_ring *rx_ring)
2052 {
2053         struct pci_dev *pdev = adapter->pdev;
2054
2055         e1000_clean_rx_ring(adapter, rx_ring);
2056
2057         vfree(rx_ring->buffer_info);
2058         rx_ring->buffer_info = NULL;
2059
2060         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2061                           rx_ring->dma);
2062
2063         rx_ring->desc = NULL;
2064 }
2065
2066 /**
2067  * e1000_free_all_rx_resources - Free Rx Resources for All Queues
2068  * @adapter: board private structure
2069  *
2070  * Free all receive software resources
2071  **/
2072 void e1000_free_all_rx_resources(struct e1000_adapter *adapter)
2073 {
2074         int i;
2075
2076         for (i = 0; i < adapter->num_rx_queues; i++)
2077                 e1000_free_rx_resources(adapter, &adapter->rx_ring[i]);
2078 }
2079
2080 #define E1000_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
2081 static unsigned int e1000_frag_len(const struct e1000_adapter *a)
2082 {
2083         return SKB_DATA_ALIGN(a->rx_buffer_len + E1000_HEADROOM) +
2084                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2085 }
2086
2087 static void *e1000_alloc_frag(const struct e1000_adapter *a)
2088 {
2089         unsigned int len = e1000_frag_len(a);
2090         u8 *data = netdev_alloc_frag(len);
2091
2092         if (likely(data))
2093                 data += E1000_HEADROOM;
2094         return data;
2095 }
2096
2097 /**
2098  * e1000_clean_rx_ring - Free Rx Buffers per Queue
2099  * @adapter: board private structure
2100  * @rx_ring: ring to free buffers from
2101  **/
2102 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
2103                                 struct e1000_rx_ring *rx_ring)
2104 {
2105         struct e1000_hw *hw = &adapter->hw;
2106         struct e1000_rx_buffer *buffer_info;
2107         struct pci_dev *pdev = adapter->pdev;
2108         unsigned long size;
2109         unsigned int i;
2110
2111         /* Free all the Rx netfrags */
2112         for (i = 0; i < rx_ring->count; i++) {
2113                 buffer_info = &rx_ring->buffer_info[i];
2114                 if (adapter->clean_rx == e1000_clean_rx_irq) {
2115                         if (buffer_info->dma)
2116                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
2117                                                  adapter->rx_buffer_len,
2118                                                  DMA_FROM_DEVICE);
2119                         if (buffer_info->rxbuf.data) {
2120                                 skb_free_frag(buffer_info->rxbuf.data);
2121                                 buffer_info->rxbuf.data = NULL;
2122                         }
2123                 } else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) {
2124                         if (buffer_info->dma)
2125                                 dma_unmap_page(&pdev->dev, buffer_info->dma,
2126                                                adapter->rx_buffer_len,
2127                                                DMA_FROM_DEVICE);
2128                         if (buffer_info->rxbuf.page) {
2129                                 put_page(buffer_info->rxbuf.page);
2130                                 buffer_info->rxbuf.page = NULL;
2131                         }
2132                 }
2133
2134                 buffer_info->dma = 0;
2135         }
2136
2137         /* there also may be some cached data from a chained receive */
2138         napi_free_frags(&adapter->napi);
2139         rx_ring->rx_skb_top = NULL;
2140
2141         size = sizeof(struct e1000_rx_buffer) * rx_ring->count;
2142         memset(rx_ring->buffer_info, 0, size);
2143
2144         /* Zero out the descriptor ring */
2145         memset(rx_ring->desc, 0, rx_ring->size);
2146
2147         rx_ring->next_to_clean = 0;
2148         rx_ring->next_to_use = 0;
2149
2150         writel(0, hw->hw_addr + rx_ring->rdh);
2151         writel(0, hw->hw_addr + rx_ring->rdt);
2152 }
2153
2154 /**
2155  * e1000_clean_all_rx_rings - Free Rx Buffers for all queues
2156  * @adapter: board private structure
2157  **/
2158 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter)
2159 {
2160         int i;
2161
2162         for (i = 0; i < adapter->num_rx_queues; i++)
2163                 e1000_clean_rx_ring(adapter, &adapter->rx_ring[i]);
2164 }
2165
2166 /* The 82542 2.0 (revision 2) needs to have the receive unit in reset
2167  * and memory write and invalidate disabled for certain operations
2168  */
2169 static void e1000_enter_82542_rst(struct e1000_adapter *adapter)
2170 {
2171         struct e1000_hw *hw = &adapter->hw;
2172         struct net_device *netdev = adapter->netdev;
2173         u32 rctl;
2174
2175         e1000_pci_clear_mwi(hw);
2176
2177         rctl = er32(RCTL);
2178         rctl |= E1000_RCTL_RST;
2179         ew32(RCTL, rctl);
2180         E1000_WRITE_FLUSH();
2181         mdelay(5);
2182
2183         if (netif_running(netdev))
2184                 e1000_clean_all_rx_rings(adapter);
2185 }
2186
2187 static void e1000_leave_82542_rst(struct e1000_adapter *adapter)
2188 {
2189         struct e1000_hw *hw = &adapter->hw;
2190         struct net_device *netdev = adapter->netdev;
2191         u32 rctl;
2192
2193         rctl = er32(RCTL);
2194         rctl &= ~E1000_RCTL_RST;
2195         ew32(RCTL, rctl);
2196         E1000_WRITE_FLUSH();
2197         mdelay(5);
2198
2199         if (hw->pci_cmd_word & PCI_COMMAND_INVALIDATE)
2200                 e1000_pci_set_mwi(hw);
2201
2202         if (netif_running(netdev)) {
2203                 /* No need to loop, because 82542 supports only 1 queue */
2204                 struct e1000_rx_ring *ring = &adapter->rx_ring[0];
2205                 e1000_configure_rx(adapter);
2206                 adapter->alloc_rx_buf(adapter, ring, E1000_DESC_UNUSED(ring));
2207         }
2208 }
2209
2210 /**
2211  * e1000_set_mac - Change the Ethernet Address of the NIC
2212  * @netdev: network interface device structure
2213  * @p: pointer to an address structure
2214  *
2215  * Returns 0 on success, negative on failure
2216  **/
2217 static int e1000_set_mac(struct net_device *netdev, void *p)
2218 {
2219         struct e1000_adapter *adapter = netdev_priv(netdev);
2220         struct e1000_hw *hw = &adapter->hw;
2221         struct sockaddr *addr = p;
2222
2223         if (!is_valid_ether_addr(addr->sa_data))
2224                 return -EADDRNOTAVAIL;
2225
2226         /* 82542 2.0 needs to be in reset to write receive address registers */
2227
2228         if (hw->mac_type == e1000_82542_rev2_0)
2229                 e1000_enter_82542_rst(adapter);
2230
2231         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2232         memcpy(hw->mac_addr, addr->sa_data, netdev->addr_len);
2233
2234         e1000_rar_set(hw, hw->mac_addr, 0);
2235
2236         if (hw->mac_type == e1000_82542_rev2_0)
2237                 e1000_leave_82542_rst(adapter);
2238
2239         return 0;
2240 }
2241
2242 /**
2243  * e1000_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2244  * @netdev: network interface device structure
2245  *
2246  * The set_rx_mode entry point is called whenever the unicast or multicast
2247  * address lists or the network interface flags are updated. This routine is
2248  * responsible for configuring the hardware for proper unicast, multicast,
2249  * promiscuous mode, and all-multi behavior.
2250  **/
2251 static void e1000_set_rx_mode(struct net_device *netdev)
2252 {
2253         struct e1000_adapter *adapter = netdev_priv(netdev);
2254         struct e1000_hw *hw = &adapter->hw;
2255         struct netdev_hw_addr *ha;
2256         bool use_uc = false;
2257         u32 rctl;
2258         u32 hash_value;
2259         int i, rar_entries = E1000_RAR_ENTRIES;
2260         int mta_reg_count = E1000_NUM_MTA_REGISTERS;
2261         u32 *mcarray = kcalloc(mta_reg_count, sizeof(u32), GFP_ATOMIC);
2262
2263         if (!mcarray)
2264                 return;
2265
2266         /* Check for Promiscuous and All Multicast modes */
2267
2268         rctl = er32(RCTL);
2269
2270         if (netdev->flags & IFF_PROMISC) {
2271                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2272                 rctl &= ~E1000_RCTL_VFE;
2273         } else {
2274                 if (netdev->flags & IFF_ALLMULTI)
2275                         rctl |= E1000_RCTL_MPE;
2276                 else
2277                         rctl &= ~E1000_RCTL_MPE;
2278                 /* Enable VLAN filter if there is a VLAN */
2279                 if (e1000_vlan_used(adapter))
2280                         rctl |= E1000_RCTL_VFE;
2281         }
2282
2283         if (netdev_uc_count(netdev) > rar_entries - 1) {
2284                 rctl |= E1000_RCTL_UPE;
2285         } else if (!(netdev->flags & IFF_PROMISC)) {
2286                 rctl &= ~E1000_RCTL_UPE;
2287                 use_uc = true;
2288         }
2289
2290         ew32(RCTL, rctl);
2291
2292         /* 82542 2.0 needs to be in reset to write receive address registers */
2293
2294         if (hw->mac_type == e1000_82542_rev2_0)
2295                 e1000_enter_82542_rst(adapter);
2296
2297         /* load the first 14 addresses into the exact filters 1-14. Unicast
2298          * addresses take precedence to avoid disabling unicast filtering
2299          * when possible.
2300          *
2301          * RAR 0 is used for the station MAC address
2302          * if there are not 14 addresses, go ahead and clear the filters
2303          */
2304         i = 1;
2305         if (use_uc)
2306                 netdev_for_each_uc_addr(ha, netdev) {
2307                         if (i == rar_entries)
2308                                 break;
2309                         e1000_rar_set(hw, ha->addr, i++);
2310                 }
2311
2312         netdev_for_each_mc_addr(ha, netdev) {
2313                 if (i == rar_entries) {
2314                         /* load any remaining addresses into the hash table */
2315                         u32 hash_reg, hash_bit, mta;
2316                         hash_value = e1000_hash_mc_addr(hw, ha->addr);
2317                         hash_reg = (hash_value >> 5) & 0x7F;
2318                         hash_bit = hash_value & 0x1F;
2319                         mta = (1 << hash_bit);
2320                         mcarray[hash_reg] |= mta;
2321                 } else {
2322                         e1000_rar_set(hw, ha->addr, i++);
2323                 }
2324         }
2325
2326         for (; i < rar_entries; i++) {
2327                 E1000_WRITE_REG_ARRAY(hw, RA, i << 1, 0);
2328                 E1000_WRITE_FLUSH();
2329                 E1000_WRITE_REG_ARRAY(hw, RA, (i << 1) + 1, 0);
2330                 E1000_WRITE_FLUSH();
2331         }
2332
2333         /* write the hash table completely, write from bottom to avoid
2334          * both stupid write combining chipsets, and flushing each write
2335          */
2336         for (i = mta_reg_count - 1; i >= 0 ; i--) {
2337                 /* If we are on an 82544 has an errata where writing odd
2338                  * offsets overwrites the previous even offset, but writing
2339                  * backwards over the range solves the issue by always
2340                  * writing the odd offset first
2341                  */
2342                 E1000_WRITE_REG_ARRAY(hw, MTA, i, mcarray[i]);
2343         }
2344         E1000_WRITE_FLUSH();
2345
2346         if (hw->mac_type == e1000_82542_rev2_0)
2347                 e1000_leave_82542_rst(adapter);
2348
2349         kfree(mcarray);
2350 }
2351
2352 /**
2353  * e1000_update_phy_info_task - get phy info
2354  * @work: work struct contained inside adapter struct
2355  *
2356  * Need to wait a few seconds after link up to get diagnostic information from
2357  * the phy
2358  */
2359 static void e1000_update_phy_info_task(struct work_struct *work)
2360 {
2361         struct e1000_adapter *adapter = container_of(work,
2362                                                      struct e1000_adapter,
2363                                                      phy_info_task.work);
2364
2365         e1000_phy_get_info(&adapter->hw, &adapter->phy_info);
2366 }
2367
2368 /**
2369  * e1000_82547_tx_fifo_stall_task - task to complete work
2370  * @work: work struct contained inside adapter struct
2371  **/
2372 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work)
2373 {
2374         struct e1000_adapter *adapter = container_of(work,
2375                                                      struct e1000_adapter,
2376                                                      fifo_stall_task.work);
2377         struct e1000_hw *hw = &adapter->hw;
2378         struct net_device *netdev = adapter->netdev;
2379         u32 tctl;
2380
2381         if (atomic_read(&adapter->tx_fifo_stall)) {
2382                 if ((er32(TDT) == er32(TDH)) &&
2383                    (er32(TDFT) == er32(TDFH)) &&
2384                    (er32(TDFTS) == er32(TDFHS))) {
2385                         tctl = er32(TCTL);
2386                         ew32(TCTL, tctl & ~E1000_TCTL_EN);
2387                         ew32(TDFT, adapter->tx_head_addr);
2388                         ew32(TDFH, adapter->tx_head_addr);
2389                         ew32(TDFTS, adapter->tx_head_addr);
2390                         ew32(TDFHS, adapter->tx_head_addr);
2391                         ew32(TCTL, tctl);
2392                         E1000_WRITE_FLUSH();
2393
2394                         adapter->tx_fifo_head = 0;
2395                         atomic_set(&adapter->tx_fifo_stall, 0);
2396                         netif_wake_queue(netdev);
2397                 } else if (!test_bit(__E1000_DOWN, &adapter->flags)) {
2398                         schedule_delayed_work(&adapter->fifo_stall_task, 1);
2399                 }
2400         }
2401 }
2402
2403 bool e1000_has_link(struct e1000_adapter *adapter)
2404 {
2405         struct e1000_hw *hw = &adapter->hw;
2406         bool link_active = false;
2407
2408         /* get_link_status is set on LSC (link status) interrupt or rx
2409          * sequence error interrupt (except on intel ce4100).
2410          * get_link_status will stay false until the
2411          * e1000_check_for_link establishes link for copper adapters
2412          * ONLY
2413          */
2414         switch (hw->media_type) {
2415         case e1000_media_type_copper:
2416                 if (hw->mac_type == e1000_ce4100)
2417                         hw->get_link_status = 1;
2418                 if (hw->get_link_status) {
2419                         e1000_check_for_link(hw);
2420                         link_active = !hw->get_link_status;
2421                 } else {
2422                         link_active = true;
2423                 }
2424                 break;
2425         case e1000_media_type_fiber:
2426                 e1000_check_for_link(hw);
2427                 link_active = !!(er32(STATUS) & E1000_STATUS_LU);
2428                 break;
2429         case e1000_media_type_internal_serdes:
2430                 e1000_check_for_link(hw);
2431                 link_active = hw->serdes_has_link;
2432                 break;
2433         default:
2434                 break;
2435         }
2436
2437         return link_active;
2438 }
2439
2440 /**
2441  * e1000_watchdog - work function
2442  * @work: work struct contained inside adapter struct
2443  **/
2444 static void e1000_watchdog(struct work_struct *work)
2445 {
2446         struct e1000_adapter *adapter = container_of(work,
2447                                                      struct e1000_adapter,
2448                                                      watchdog_task.work);
2449         struct e1000_hw *hw = &adapter->hw;
2450         struct net_device *netdev = adapter->netdev;
2451         struct e1000_tx_ring *txdr = adapter->tx_ring;
2452         u32 link, tctl;
2453
2454         link = e1000_has_link(adapter);
2455         if ((netif_carrier_ok(netdev)) && link)
2456                 goto link_up;
2457
2458         if (link) {
2459                 if (!netif_carrier_ok(netdev)) {
2460                         u32 ctrl;
2461                         bool txb2b = true;
2462                         /* update snapshot of PHY registers on LSC */
2463                         e1000_get_speed_and_duplex(hw,
2464                                                    &adapter->link_speed,
2465                                                    &adapter->link_duplex);
2466
2467                         ctrl = er32(CTRL);
2468                         pr_info("%s NIC Link is Up %d Mbps %s, "
2469                                 "Flow Control: %s\n",
2470                                 netdev->name,
2471                                 adapter->link_speed,
2472                                 adapter->link_duplex == FULL_DUPLEX ?
2473                                 "Full Duplex" : "Half Duplex",
2474                                 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2475                                 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2476                                 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2477                                 E1000_CTRL_TFCE) ? "TX" : "None")));
2478
2479                         /* adjust timeout factor according to speed/duplex */
2480                         adapter->tx_timeout_factor = 1;
2481                         switch (adapter->link_speed) {
2482                         case SPEED_10:
2483                                 txb2b = false;
2484                                 adapter->tx_timeout_factor = 16;
2485                                 break;
2486                         case SPEED_100:
2487                                 txb2b = false;
2488                                 /* maybe add some timeout factor ? */
2489                                 break;
2490                         }
2491
2492                         /* enable transmits in the hardware */
2493                         tctl = er32(TCTL);
2494                         tctl |= E1000_TCTL_EN;
2495                         ew32(TCTL, tctl);
2496
2497                         netif_carrier_on(netdev);
2498                         if (!test_bit(__E1000_DOWN, &adapter->flags))
2499                                 schedule_delayed_work(&adapter->phy_info_task,
2500                                                       2 * HZ);
2501                         adapter->smartspeed = 0;
2502                 }
2503         } else {
2504                 if (netif_carrier_ok(netdev)) {
2505                         adapter->link_speed = 0;
2506                         adapter->link_duplex = 0;
2507                         pr_info("%s NIC Link is Down\n",
2508                                 netdev->name);
2509                         netif_carrier_off(netdev);
2510
2511                         if (!test_bit(__E1000_DOWN, &adapter->flags))
2512                                 schedule_delayed_work(&adapter->phy_info_task,
2513                                                       2 * HZ);
2514                 }
2515
2516                 e1000_smartspeed(adapter);
2517         }
2518
2519 link_up:
2520         e1000_update_stats(adapter);
2521
2522         hw->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
2523         adapter->tpt_old = adapter->stats.tpt;
2524         hw->collision_delta = adapter->stats.colc - adapter->colc_old;
2525         adapter->colc_old = adapter->stats.colc;
2526
2527         adapter->gorcl = adapter->stats.gorcl - adapter->gorcl_old;
2528         adapter->gorcl_old = adapter->stats.gorcl;
2529         adapter->gotcl = adapter->stats.gotcl - adapter->gotcl_old;
2530         adapter->gotcl_old = adapter->stats.gotcl;
2531
2532         e1000_update_adaptive(hw);
2533
2534         if (!netif_carrier_ok(netdev)) {
2535                 if (E1000_DESC_UNUSED(txdr) + 1 < txdr->count) {
2536                         /* We've lost link, so the controller stops DMA,
2537                          * but we've got queued Tx work that's never going
2538                          * to get done, so reset controller to flush Tx.
2539                          * (Do the reset outside of interrupt context).
2540                          */
2541                         adapter->tx_timeout_count++;
2542                         schedule_work(&adapter->reset_task);
2543                         /* exit immediately since reset is imminent */
2544                         return;
2545                 }
2546         }
2547
2548         /* Simple mode for Interrupt Throttle Rate (ITR) */
2549         if (hw->mac_type >= e1000_82540 && adapter->itr_setting == 4) {
2550                 /* Symmetric Tx/Rx gets a reduced ITR=2000;
2551                  * Total asymmetrical Tx or Rx gets ITR=8000;
2552                  * everyone else is between 2000-8000.
2553                  */
2554                 u32 goc = (adapter->gotcl + adapter->gorcl) / 10000;
2555                 u32 dif = (adapter->gotcl > adapter->gorcl ?
2556                             adapter->gotcl - adapter->gorcl :
2557                             adapter->gorcl - adapter->gotcl) / 10000;
2558                 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
2559
2560                 ew32(ITR, 1000000000 / (itr * 256));
2561         }
2562
2563         /* Cause software interrupt to ensure rx ring is cleaned */
2564         ew32(ICS, E1000_ICS_RXDMT0);
2565
2566         /* Force detection of hung controller every watchdog period */
2567         adapter->detect_tx_hung = true;
2568
2569         /* Reschedule the task */
2570         if (!test_bit(__E1000_DOWN, &adapter->flags))
2571                 schedule_delayed_work(&adapter->watchdog_task, 2 * HZ);
2572 }
2573
2574 enum latency_range {
2575         lowest_latency = 0,
2576         low_latency = 1,
2577         bulk_latency = 2,
2578         latency_invalid = 255
2579 };
2580
2581 /**
2582  * e1000_update_itr - update the dynamic ITR value based on statistics
2583  * @adapter: pointer to adapter
2584  * @itr_setting: current adapter->itr
2585  * @packets: the number of packets during this measurement interval
2586  * @bytes: the number of bytes during this measurement interval
2587  *
2588  *      Stores a new ITR value based on packets and byte
2589  *      counts during the last interrupt.  The advantage of per interrupt
2590  *      computation is faster updates and more accurate ITR for the current
2591  *      traffic pattern.  Constants in this function were computed
2592  *      based on theoretical maximum wire speed and thresholds were set based
2593  *      on testing data as well as attempting to minimize response time
2594  *      while increasing bulk throughput.
2595  *      this functionality is controlled by the InterruptThrottleRate module
2596  *      parameter (see e1000_param.c)
2597  **/
2598 static unsigned int e1000_update_itr(struct e1000_adapter *adapter,
2599                                      u16 itr_setting, int packets, int bytes)
2600 {
2601         unsigned int retval = itr_setting;
2602         struct e1000_hw *hw = &adapter->hw;
2603
2604         if (unlikely(hw->mac_type < e1000_82540))
2605                 goto update_itr_done;
2606
2607         if (packets == 0)
2608                 goto update_itr_done;
2609
2610         switch (itr_setting) {
2611         case lowest_latency:
2612                 /* jumbo frames get bulk treatment*/
2613                 if (bytes/packets > 8000)
2614                         retval = bulk_latency;
2615                 else if ((packets < 5) && (bytes > 512))
2616                         retval = low_latency;
2617                 break;
2618         case low_latency:  /* 50 usec aka 20000 ints/s */
2619                 if (bytes > 10000) {
2620                         /* jumbo frames need bulk latency setting */
2621                         if (bytes/packets > 8000)
2622                                 retval = bulk_latency;
2623                         else if ((packets < 10) || ((bytes/packets) > 1200))
2624                                 retval = bulk_latency;
2625                         else if ((packets > 35))
2626                                 retval = lowest_latency;
2627                 } else if (bytes/packets > 2000)
2628                         retval = bulk_latency;
2629                 else if (packets <= 2 && bytes < 512)
2630                         retval = lowest_latency;
2631                 break;
2632         case bulk_latency: /* 250 usec aka 4000 ints/s */
2633                 if (bytes > 25000) {
2634                         if (packets > 35)
2635                                 retval = low_latency;
2636                 } else if (bytes < 6000) {
2637                         retval = low_latency;
2638                 }
2639                 break;
2640         }
2641
2642 update_itr_done:
2643         return retval;
2644 }
2645
2646 static void e1000_set_itr(struct e1000_adapter *adapter)
2647 {
2648         struct e1000_hw *hw = &adapter->hw;
2649         u16 current_itr;
2650         u32 new_itr = adapter->itr;
2651
2652         if (unlikely(hw->mac_type < e1000_82540))
2653                 return;
2654
2655         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2656         if (unlikely(adapter->link_speed != SPEED_1000)) {
2657                 current_itr = 0;
2658                 new_itr = 4000;
2659                 goto set_itr_now;
2660         }
2661
2662         adapter->tx_itr = e1000_update_itr(adapter, adapter->tx_itr,
2663                                            adapter->total_tx_packets,
2664                                            adapter->total_tx_bytes);
2665         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2666         if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2667                 adapter->tx_itr = low_latency;
2668
2669         adapter->rx_itr = e1000_update_itr(adapter, adapter->rx_itr,
2670                                            adapter->total_rx_packets,
2671                                            adapter->total_rx_bytes);
2672         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2673         if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2674                 adapter->rx_itr = low_latency;
2675
2676         current_itr = max(adapter->rx_itr, adapter->tx_itr);
2677
2678         switch (current_itr) {
2679         /* counts and packets in update_itr are dependent on these numbers */
2680         case lowest_latency:
2681                 new_itr = 70000;
2682                 break;
2683         case low_latency:
2684                 new_itr = 20000; /* aka hwitr = ~200 */
2685                 break;
2686         case bulk_latency:
2687                 new_itr = 4000;
2688                 break;
2689         default:
2690                 break;
2691         }
2692
2693 set_itr_now:
2694         if (new_itr != adapter->itr) {
2695                 /* this attempts to bias the interrupt rate towards Bulk
2696                  * by adding intermediate steps when interrupt rate is
2697                  * increasing
2698                  */
2699                 new_itr = new_itr > adapter->itr ?
2700                           min(adapter->itr + (new_itr >> 2), new_itr) :
2701                           new_itr;
2702                 adapter->itr = new_itr;
2703                 ew32(ITR, 1000000000 / (new_itr * 256));
2704         }
2705 }
2706
2707 #define E1000_TX_FLAGS_CSUM             0x00000001
2708 #define E1000_TX_FLAGS_VLAN             0x00000002
2709 #define E1000_TX_FLAGS_TSO              0x00000004
2710 #define E1000_TX_FLAGS_IPV4             0x00000008
2711 #define E1000_TX_FLAGS_NO_FCS           0x00000010
2712 #define E1000_TX_FLAGS_VLAN_MASK        0xffff0000
2713 #define E1000_TX_FLAGS_VLAN_SHIFT       16
2714
2715 static int e1000_tso(struct e1000_adapter *adapter,
2716                      struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2717                      __be16 protocol)
2718 {
2719         struct e1000_context_desc *context_desc;
2720         struct e1000_tx_buffer *buffer_info;
2721         unsigned int i;
2722         u32 cmd_length = 0;
2723         u16 ipcse = 0, tucse, mss;
2724         u8 ipcss, ipcso, tucss, tucso, hdr_len;
2725
2726         if (skb_is_gso(skb)) {
2727                 int err;
2728
2729                 err = skb_cow_head(skb, 0);
2730                 if (err < 0)
2731                         return err;
2732
2733                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2734                 mss = skb_shinfo(skb)->gso_size;
2735                 if (protocol == htons(ETH_P_IP)) {
2736                         struct iphdr *iph = ip_hdr(skb);
2737                         iph->tot_len = 0;
2738                         iph->check = 0;
2739                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2740                                                                  iph->daddr, 0,
2741                                                                  IPPROTO_TCP,
2742                                                                  0);
2743                         cmd_length = E1000_TXD_CMD_IP;
2744                         ipcse = skb_transport_offset(skb) - 1;
2745                 } else if (skb_is_gso_v6(skb)) {
2746                         ipv6_hdr(skb)->payload_len = 0;
2747                         tcp_hdr(skb)->check =
2748                                 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2749                                                  &ipv6_hdr(skb)->daddr,
2750                                                  0, IPPROTO_TCP, 0);
2751                         ipcse = 0;
2752                 }
2753                 ipcss = skb_network_offset(skb);
2754                 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
2755                 tucss = skb_transport_offset(skb);
2756                 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
2757                 tucse = 0;
2758
2759                 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
2760                                E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
2761
2762                 i = tx_ring->next_to_use;
2763                 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2764                 buffer_info = &tx_ring->buffer_info[i];
2765
2766                 context_desc->lower_setup.ip_fields.ipcss  = ipcss;
2767                 context_desc->lower_setup.ip_fields.ipcso  = ipcso;
2768                 context_desc->lower_setup.ip_fields.ipcse  = cpu_to_le16(ipcse);
2769                 context_desc->upper_setup.tcp_fields.tucss = tucss;
2770                 context_desc->upper_setup.tcp_fields.tucso = tucso;
2771                 context_desc->upper_setup.tcp_fields.tucse = cpu_to_le16(tucse);
2772                 context_desc->tcp_seg_setup.fields.mss     = cpu_to_le16(mss);
2773                 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
2774                 context_desc->cmd_and_length = cpu_to_le32(cmd_length);
2775
2776                 buffer_info->time_stamp = jiffies;
2777                 buffer_info->next_to_watch = i;
2778
2779                 if (++i == tx_ring->count)
2780                         i = 0;
2781
2782                 tx_ring->next_to_use = i;
2783
2784                 return true;
2785         }
2786         return false;
2787 }
2788
2789 static bool e1000_tx_csum(struct e1000_adapter *adapter,
2790                           struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2791                           __be16 protocol)
2792 {
2793         struct e1000_context_desc *context_desc;
2794         struct e1000_tx_buffer *buffer_info;
2795         unsigned int i;
2796         u8 css;
2797         u32 cmd_len = E1000_TXD_CMD_DEXT;
2798
2799         if (skb->ip_summed != CHECKSUM_PARTIAL)
2800                 return false;
2801
2802         switch (protocol) {
2803         case cpu_to_be16(ETH_P_IP):
2804                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2805                         cmd_len |= E1000_TXD_CMD_TCP;
2806                 break;
2807         case cpu_to_be16(ETH_P_IPV6):
2808                 /* XXX not handling all IPV6 headers */
2809                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2810                         cmd_len |= E1000_TXD_CMD_TCP;
2811                 break;
2812         default:
2813                 if (unlikely(net_ratelimit()))
2814                         e_warn(drv, "checksum_partial proto=%x!\n",
2815                                skb->protocol);
2816                 break;
2817         }
2818
2819         css = skb_checksum_start_offset(skb);
2820
2821         i = tx_ring->next_to_use;
2822         buffer_info = &tx_ring->buffer_info[i];
2823         context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2824
2825         context_desc->lower_setup.ip_config = 0;
2826         context_desc->upper_setup.tcp_fields.tucss = css;
2827         context_desc->upper_setup.tcp_fields.tucso =
2828                 css + skb->csum_offset;
2829         context_desc->upper_setup.tcp_fields.tucse = 0;
2830         context_desc->tcp_seg_setup.data = 0;
2831         context_desc->cmd_and_length = cpu_to_le32(cmd_len);
2832
2833         buffer_info->time_stamp = jiffies;
2834         buffer_info->next_to_watch = i;
2835
2836         if (unlikely(++i == tx_ring->count))
2837                 i = 0;
2838
2839         tx_ring->next_to_use = i;
2840
2841         return true;
2842 }
2843
2844 #define E1000_MAX_TXD_PWR       12
2845 #define E1000_MAX_DATA_PER_TXD  (1<<E1000_MAX_TXD_PWR)
2846
2847 static int e1000_tx_map(struct e1000_adapter *adapter,
2848                         struct e1000_tx_ring *tx_ring,
2849                         struct sk_buff *skb, unsigned int first,
2850                         unsigned int max_per_txd, unsigned int nr_frags,
2851                         unsigned int mss)
2852 {
2853         struct e1000_hw *hw = &adapter->hw;
2854         struct pci_dev *pdev = adapter->pdev;
2855         struct e1000_tx_buffer *buffer_info;
2856         unsigned int len = skb_headlen(skb);
2857         unsigned int offset = 0, size, count = 0, i;
2858         unsigned int f, bytecount, segs;
2859
2860         i = tx_ring->next_to_use;
2861
2862         while (len) {
2863                 buffer_info = &tx_ring->buffer_info[i];
2864                 size = min(len, max_per_txd);
2865                 /* Workaround for Controller erratum --
2866                  * descriptor for non-tso packet in a linear SKB that follows a
2867                  * tso gets written back prematurely before the data is fully
2868                  * DMA'd to the controller
2869                  */
2870                 if (!skb->data_len && tx_ring->last_tx_tso &&
2871                     !skb_is_gso(skb)) {
2872                         tx_ring->last_tx_tso = false;
2873                         size -= 4;
2874                 }
2875
2876                 /* Workaround for premature desc write-backs
2877                  * in TSO mode.  Append 4-byte sentinel desc
2878                  */
2879                 if (unlikely(mss && !nr_frags && size == len && size > 8))
2880                         size -= 4;
2881                 /* work-around for errata 10 and it applies
2882                  * to all controllers in PCI-X mode
2883                  * The fix is to make sure that the first descriptor of a
2884                  * packet is smaller than 2048 - 16 - 16 (or 2016) bytes
2885                  */
2886                 if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
2887                              (size > 2015) && count == 0))
2888                         size = 2015;
2889
2890                 /* Workaround for potential 82544 hang in PCI-X.  Avoid
2891                  * terminating buffers within evenly-aligned dwords.
2892                  */
2893                 if (unlikely(adapter->pcix_82544 &&
2894                    !((unsigned long)(skb->data + offset + size - 1) & 4) &&
2895                    size > 4))
2896                         size -= 4;
2897
2898                 buffer_info->length = size;
2899                 /* set time_stamp *before* dma to help avoid a possible race */
2900                 buffer_info->time_stamp = jiffies;
2901                 buffer_info->mapped_as_page = false;
2902                 buffer_info->dma = dma_map_single(&pdev->dev,
2903                                                   skb->data + offset,
2904                                                   size, DMA_TO_DEVICE);
2905                 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2906                         goto dma_error;
2907                 buffer_info->next_to_watch = i;
2908
2909                 len -= size;
2910                 offset += size;
2911                 count++;
2912                 if (len) {
2913                         i++;
2914                         if (unlikely(i == tx_ring->count))
2915                                 i = 0;
2916                 }
2917         }
2918
2919         for (f = 0; f < nr_frags; f++) {
2920                 const struct skb_frag_struct *frag;
2921
2922                 frag = &skb_shinfo(skb)->frags[f];
2923                 len = skb_frag_size(frag);
2924                 offset = 0;
2925
2926                 while (len) {
2927                         unsigned long bufend;
2928                         i++;
2929                         if (unlikely(i == tx_ring->count))
2930                                 i = 0;
2931
2932                         buffer_info = &tx_ring->buffer_info[i];
2933                         size = min(len, max_per_txd);
2934                         /* Workaround for premature desc write-backs
2935                          * in TSO mode.  Append 4-byte sentinel desc
2936                          */
2937                         if (unlikely(mss && f == (nr_frags-1) &&
2938                             size == len && size > 8))
2939                                 size -= 4;
2940                         /* Workaround for potential 82544 hang in PCI-X.
2941                          * Avoid terminating buffers within evenly-aligned
2942                          * dwords.
2943                          */
2944                         bufend = (unsigned long)
2945                                 page_to_phys(skb_frag_page(frag));
2946                         bufend += offset + size - 1;
2947                         if (unlikely(adapter->pcix_82544 &&
2948                                      !(bufend & 4) &&
2949                                      size > 4))
2950                                 size -= 4;
2951
2952                         buffer_info->length = size;
2953                         buffer_info->time_stamp = jiffies;
2954                         buffer_info->mapped_as_page = true;
2955                         buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
2956                                                 offset, size, DMA_TO_DEVICE);
2957                         if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2958                                 goto dma_error;
2959                         buffer_info->next_to_watch = i;
2960
2961                         len -= size;
2962                         offset += size;
2963                         count++;
2964                 }
2965         }
2966
2967         segs = skb_shinfo(skb)->gso_segs ?: 1;
2968         /* multiply data chunks by size of headers */
2969         bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
2970
2971         tx_ring->buffer_info[i].skb = skb;
2972         tx_ring->buffer_info[i].segs = segs;
2973         tx_ring->buffer_info[i].bytecount = bytecount;
2974         tx_ring->buffer_info[first].next_to_watch = i;
2975
2976         return count;
2977
2978 dma_error:
2979         dev_err(&pdev->dev, "TX DMA map failed\n");
2980         buffer_info->dma = 0;
2981         if (count)
2982                 count--;
2983
2984         while (count--) {
2985                 if (i == 0)
2986                         i += tx_ring->count;
2987                 i--;
2988                 buffer_info = &tx_ring->buffer_info[i];
2989                 e1000_unmap_and_free_tx_resource(adapter, buffer_info);
2990         }
2991
2992         return 0;
2993 }
2994
2995 static void e1000_tx_queue(struct e1000_adapter *adapter,
2996                            struct e1000_tx_ring *tx_ring, int tx_flags,
2997                            int count)
2998 {
2999         struct e1000_tx_desc *tx_desc = NULL;
3000         struct e1000_tx_buffer *buffer_info;
3001         u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
3002         unsigned int i;
3003
3004         if (likely(tx_flags & E1000_TX_FLAGS_TSO)) {
3005                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
3006                              E1000_TXD_CMD_TSE;
3007                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3008
3009                 if (likely(tx_flags & E1000_TX_FLAGS_IPV4))
3010                         txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3011         }
3012
3013         if (likely(tx_flags & E1000_TX_FLAGS_CSUM)) {
3014                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3015                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3016         }
3017
3018         if (unlikely(tx_flags & E1000_TX_FLAGS_VLAN)) {
3019                 txd_lower |= E1000_TXD_CMD_VLE;
3020                 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
3021         }
3022
3023         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
3024                 txd_lower &= ~(E1000_TXD_CMD_IFCS);
3025
3026         i = tx_ring->next_to_use;
3027
3028         while (count--) {
3029                 buffer_info = &tx_ring->buffer_info[i];
3030                 tx_desc = E1000_TX_DESC(*tx_ring, i);
3031                 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
3032                 tx_desc->lower.data =
3033                         cpu_to_le32(txd_lower | buffer_info->length);
3034                 tx_desc->upper.data = cpu_to_le32(txd_upper);
3035                 if (unlikely(++i == tx_ring->count))
3036                         i = 0;
3037         }
3038
3039         tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
3040
3041         /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */
3042         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
3043                 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));
3044
3045         /* Force memory writes to complete before letting h/w
3046          * know there are new descriptors to fetch.  (Only
3047          * applicable for weak-ordered memory model archs,
3048          * such as IA-64).
3049          */
3050         wmb();
3051
3052         tx_ring->next_to_use = i;
3053 }
3054
3055 /* 82547 workaround to avoid controller hang in half-duplex environment.
3056  * The workaround is to avoid queuing a large packet that would span
3057  * the internal Tx FIFO ring boundary by notifying the stack to resend
3058  * the packet at a later time.  This gives the Tx FIFO an opportunity to
3059  * flush all packets.  When that occurs, we reset the Tx FIFO pointers
3060  * to the beginning of the Tx FIFO.
3061  */
3062
3063 #define E1000_FIFO_HDR                  0x10
3064 #define E1000_82547_PAD_LEN             0x3E0
3065
3066 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
3067                                        struct sk_buff *skb)
3068 {
3069         u32 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
3070         u32 skb_fifo_len = skb->len + E1000_FIFO_HDR;
3071
3072         skb_fifo_len = ALIGN(skb_fifo_len, E1000_FIFO_HDR);
3073
3074         if (adapter->link_duplex != HALF_DUPLEX)
3075                 goto no_fifo_stall_required;
3076
3077         if (atomic_read(&adapter->tx_fifo_stall))
3078                 return 1;
3079
3080         if (skb_fifo_len >= (E1000_82547_PAD_LEN + fifo_space)) {
3081                 atomic_set(&adapter->tx_fifo_stall, 1);
3082                 return 1;
3083         }
3084
3085 no_fifo_stall_required:
3086         adapter->tx_fifo_head += skb_fifo_len;
3087         if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
3088                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
3089         return 0;
3090 }
3091
3092 static int __e1000_maybe_stop_tx(struct net_device *netdev, int size)
3093 {
3094         struct e1000_adapter *adapter = netdev_priv(netdev);
3095         struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3096
3097         netif_stop_queue(netdev);
3098         /* Herbert's original patch had:
3099          *  smp_mb__after_netif_stop_queue();
3100          * but since that doesn't exist yet, just open code it.
3101          */
3102         smp_mb();
3103
3104         /* We need to check again in a case another CPU has just
3105          * made room available.
3106          */
3107         if (likely(E1000_DESC_UNUSED(tx_ring) < size))
3108                 return -EBUSY;
3109
3110         /* A reprieve! */
3111         netif_start_queue(netdev);
3112         ++adapter->restart_queue;
3113         return 0;
3114 }
3115
3116 static int e1000_maybe_stop_tx(struct net_device *netdev,
3117                                struct e1000_tx_ring *tx_ring, int size)
3118 {
3119         if (likely(E1000_DESC_UNUSED(tx_ring) >= size))
3120                 return 0;
3121         return __e1000_maybe_stop_tx(netdev, size);
3122 }
3123
3124 #define TXD_USE_COUNT(S, X) (((S) + ((1 << (X)) - 1)) >> (X))
3125 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
3126                                     struct net_device *netdev)
3127 {
3128         struct e1000_adapter *adapter = netdev_priv(netdev);
3129         struct e1000_hw *hw = &adapter->hw;
3130         struct e1000_tx_ring *tx_ring;
3131         unsigned int first, max_per_txd = E1000_MAX_DATA_PER_TXD;
3132         unsigned int max_txd_pwr = E1000_MAX_TXD_PWR;
3133         unsigned int tx_flags = 0;
3134         unsigned int len = skb_headlen(skb);
3135         unsigned int nr_frags;
3136         unsigned int mss;
3137         int count = 0;
3138         int tso;
3139         unsigned int f;
3140         __be16 protocol = vlan_get_protocol(skb);
3141
3142         /* This goes back to the question of how to logically map a Tx queue
3143          * to a flow.  Right now, performance is impacted slightly negatively
3144          * if using multiple Tx queues.  If the stack breaks away from a
3145          * single qdisc implementation, we can look at this again.
3146          */
3147         tx_ring = adapter->tx_ring;
3148
3149         /* On PCI/PCI-X HW, if packet size is less than ETH_ZLEN,
3150          * packets may get corrupted during padding by HW.
3151          * To WA this issue, pad all small packets manually.
3152          */
3153         if (eth_skb_pad(skb))
3154                 return NETDEV_TX_OK;
3155
3156         mss = skb_shinfo(skb)->gso_size;
3157         /* The controller does a simple calculation to
3158          * make sure there is enough room in the FIFO before
3159          * initiating the DMA for each buffer.  The calc is:
3160          * 4 = ceil(buffer len/mss).  To make sure we don't
3161          * overrun the FIFO, adjust the max buffer len if mss
3162          * drops.
3163          */
3164         if (mss) {
3165                 u8 hdr_len;
3166                 max_per_txd = min(mss << 2, max_per_txd);
3167                 max_txd_pwr = fls(max_per_txd) - 1;
3168
3169                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
3170                 if (skb->data_len && hdr_len == len) {
3171                         switch (hw->mac_type) {
3172                                 unsigned int pull_size;
3173                         case e1000_82544:
3174                                 /* Make sure we have room to chop off 4 bytes,
3175                                  * and that the end alignment will work out to
3176                                  * this hardware's requirements
3177                                  * NOTE: this is a TSO only workaround
3178                                  * if end byte alignment not correct move us
3179                                  * into the next dword
3180                                  */
3181                                 if ((unsigned long)(skb_tail_pointer(skb) - 1)
3182                                     & 4)
3183                                         break;
3184                                 /* fall through */
3185                                 pull_size = min((unsigned int)4, skb->data_len);
3186                                 if (!__pskb_pull_tail(skb, pull_size)) {
3187                                         e_err(drv, "__pskb_pull_tail "
3188                                               "failed.\n");
3189                                         dev_kfree_skb_any(skb);
3190                                         return NETDEV_TX_OK;
3191                                 }
3192                                 len = skb_headlen(skb);
3193                                 break;
3194                         default:
3195                                 /* do nothing */
3196                                 break;
3197                         }
3198                 }
3199         }
3200
3201         /* reserve a descriptor for the offload context */
3202         if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
3203                 count++;
3204         count++;
3205
3206         /* Controller Erratum workaround */
3207         if (!skb->data_len && tx_ring->last_tx_tso && !skb_is_gso(skb))
3208                 count++;
3209
3210         count += TXD_USE_COUNT(len, max_txd_pwr);
3211
3212         if (adapter->pcix_82544)
3213                 count++;
3214
3215         /* work-around for errata 10 and it applies to all controllers
3216          * in PCI-X mode, so add one more descriptor to the count
3217          */
3218         if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
3219                         (len > 2015)))
3220                 count++;
3221
3222         nr_frags = skb_shinfo(skb)->nr_frags;
3223         for (f = 0; f < nr_frags; f++)
3224                 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->frags[f]),
3225                                        max_txd_pwr);
3226         if (adapter->pcix_82544)
3227                 count += nr_frags;
3228
3229         /* need: count + 2 desc gap to keep tail from touching
3230          * head, otherwise try next time
3231          */
3232         if (unlikely(e1000_maybe_stop_tx(netdev, tx_ring, count + 2)))
3233                 return NETDEV_TX_BUSY;
3234
3235         if (unlikely((hw->mac_type == e1000_82547) &&
3236                      (e1000_82547_fifo_workaround(adapter, skb)))) {
3237                 netif_stop_queue(netdev);
3238                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3239                         schedule_delayed_work(&adapter->fifo_stall_task, 1);
3240                 return NETDEV_TX_BUSY;
3241         }
3242
3243         if (skb_vlan_tag_present(skb)) {
3244                 tx_flags |= E1000_TX_FLAGS_VLAN;
3245                 tx_flags |= (skb_vlan_tag_get(skb) <<
3246                              E1000_TX_FLAGS_VLAN_SHIFT);
3247         }
3248
3249         first = tx_ring->next_to_use;
3250
3251         tso = e1000_tso(adapter, tx_ring, skb, protocol);
3252         if (tso < 0) {
3253                 dev_kfree_skb_any(skb);
3254                 return NETDEV_TX_OK;
3255         }
3256
3257         if (likely(tso)) {
3258                 if (likely(hw->mac_type != e1000_82544))
3259                         tx_ring->last_tx_tso = true;
3260                 tx_flags |= E1000_TX_FLAGS_TSO;
3261         } else if (likely(e1000_tx_csum(adapter, tx_ring, skb, protocol)))
3262                 tx_flags |= E1000_TX_FLAGS_CSUM;
3263
3264         if (protocol == htons(ETH_P_IP))
3265                 tx_flags |= E1000_TX_FLAGS_IPV4;
3266
3267         if (unlikely(skb->no_fcs))
3268                 tx_flags |= E1000_TX_FLAGS_NO_FCS;
3269
3270         count = e1000_tx_map(adapter, tx_ring, skb, first, max_per_txd,
3271                              nr_frags, mss);
3272
3273         if (count) {
3274                 /* The descriptors needed is higher than other Intel drivers
3275                  * due to a number of workarounds.  The breakdown is below:
3276                  * Data descriptors: MAX_SKB_FRAGS + 1
3277                  * Context Descriptor: 1
3278                  * Keep head from touching tail: 2
3279                  * Workarounds: 3
3280                  */
3281                 int desc_needed = MAX_SKB_FRAGS + 7;
3282
3283                 netdev_sent_queue(netdev, skb->len);
3284                 skb_tx_timestamp(skb);
3285
3286                 e1000_tx_queue(adapter, tx_ring, tx_flags, count);
3287
3288                 /* 82544 potentially requires twice as many data descriptors
3289                  * in order to guarantee buffers don't end on evenly-aligned
3290                  * dwords
3291                  */
3292                 if (adapter->pcix_82544)
3293                         desc_needed += MAX_SKB_FRAGS + 1;
3294
3295                 /* Make sure there is space in the ring for the next send. */
3296                 e1000_maybe_stop_tx(netdev, tx_ring, desc_needed);
3297
3298                 if (!skb->xmit_more ||
3299                     netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) {
3300                         writel(tx_ring->next_to_use, hw->hw_addr + tx_ring->tdt);
3301                         /* we need this if more than one processor can write to
3302                          * our tail at a time, it synchronizes IO on IA64/Altix
3303                          * systems
3304                          */
3305                         mmiowb();
3306                 }
3307         } else {
3308                 dev_kfree_skb_any(skb);
3309                 tx_ring->buffer_info[first].time_stamp = 0;
3310                 tx_ring->next_to_use = first;
3311         }
3312
3313         return NETDEV_TX_OK;
3314 }
3315
3316 #define NUM_REGS 38 /* 1 based count */
3317 static void e1000_regdump(struct e1000_adapter *adapter)
3318 {
3319         struct e1000_hw *hw = &adapter->hw;
3320         u32 regs[NUM_REGS];
3321         u32 *regs_buff = regs;
3322         int i = 0;
3323
3324         static const char * const reg_name[] = {
3325                 "CTRL",  "STATUS",
3326                 "RCTL", "RDLEN", "RDH", "RDT", "RDTR",
3327                 "TCTL", "TDBAL", "TDBAH", "TDLEN", "TDH", "TDT",
3328                 "TIDV", "TXDCTL", "TADV", "TARC0",
3329                 "TDBAL1", "TDBAH1", "TDLEN1", "TDH1", "TDT1",
3330                 "TXDCTL1", "TARC1",
3331                 "CTRL_EXT", "ERT", "RDBAL", "RDBAH",
3332                 "TDFH", "TDFT", "TDFHS", "TDFTS", "TDFPC",
3333                 "RDFH", "RDFT", "RDFHS", "RDFTS", "RDFPC"
3334         };
3335
3336         regs_buff[0]  = er32(CTRL);
3337         regs_buff[1]  = er32(STATUS);
3338
3339         regs_buff[2]  = er32(RCTL);
3340         regs_buff[3]  = er32(RDLEN);
3341         regs_buff[4]  = er32(RDH);
3342         regs_buff[5]  = er32(RDT);
3343         regs_buff[6]  = er32(RDTR);
3344
3345         regs_buff[7]  = er32(TCTL);
3346         regs_buff[8]  = er32(TDBAL);
3347         regs_buff[9]  = er32(TDBAH);
3348         regs_buff[10] = er32(TDLEN);
3349         regs_buff[11] = er32(TDH);
3350         regs_buff[12] = er32(TDT);
3351         regs_buff[13] = er32(TIDV);
3352         regs_buff[14] = er32(TXDCTL);
3353         regs_buff[15] = er32(TADV);
3354         regs_buff[16] = er32(TARC0);
3355
3356         regs_buff[17] = er32(TDBAL1);
3357         regs_buff[18] = er32(TDBAH1);
3358         regs_buff[19] = er32(TDLEN1);
3359         regs_buff[20] = er32(TDH1);
3360         regs_buff[21] = er32(TDT1);
3361         regs_buff[22] = er32(TXDCTL1);
3362         regs_buff[23] = er32(TARC1);
3363         regs_buff[24] = er32(CTRL_EXT);
3364         regs_buff[25] = er32(ERT);
3365         regs_buff[26] = er32(RDBAL0);
3366         regs_buff[27] = er32(RDBAH0);
3367         regs_buff[28] = er32(TDFH);
3368         regs_buff[29] = er32(TDFT);
3369         regs_buff[30] = er32(TDFHS);
3370         regs_buff[31] = er32(TDFTS);
3371         regs_buff[32] = er32(TDFPC);
3372         regs_buff[33] = er32(RDFH);
3373         regs_buff[34] = er32(RDFT);
3374         regs_buff[35] = er32(RDFHS);
3375         regs_buff[36] = er32(RDFTS);
3376         regs_buff[37] = er32(RDFPC);
3377
3378         pr_info("Register dump\n");
3379         for (i = 0; i < NUM_REGS; i++)
3380                 pr_info("%-15s  %08x\n", reg_name[i], regs_buff[i]);
3381 }
3382
3383 /*
3384  * e1000_dump: Print registers, tx ring and rx ring
3385  */
3386 static void e1000_dump(struct e1000_adapter *adapter)
3387 {
3388         /* this code doesn't handle multiple rings */
3389         struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3390         struct e1000_rx_ring *rx_ring = adapter->rx_ring;
3391         int i;
3392
3393         if (!netif_msg_hw(adapter))
3394                 return;
3395
3396         /* Print Registers */
3397         e1000_regdump(adapter);
3398
3399         /* transmit dump */
3400         pr_info("TX Desc ring0 dump\n");
3401
3402         /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
3403          *
3404          * Legacy Transmit Descriptor
3405          *   +--------------------------------------------------------------+
3406          * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
3407          *   +--------------------------------------------------------------+
3408          * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
3409          *   +--------------------------------------------------------------+
3410          *   63       48 47        36 35    32 31     24 23    16 15        0
3411          *
3412          * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
3413          *   63      48 47    40 39       32 31             16 15    8 7      0
3414          *   +----------------------------------------------------------------+
3415          * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
3416          *   +----------------------------------------------------------------+
3417          * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
3418          *   +----------------------------------------------------------------+
3419          *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
3420          *
3421          * Extended Data Descriptor (DTYP=0x1)
3422          *   +----------------------------------------------------------------+
3423          * 0 |                     Buffer Address [63:0]                      |
3424          *   +----------------------------------------------------------------+
3425          * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
3426          *   +----------------------------------------------------------------+
3427          *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
3428          */
3429         pr_info("Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3430         pr_info("Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3431
3432         if (!netif_msg_tx_done(adapter))
3433                 goto rx_ring_summary;
3434
3435         for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
3436                 struct e1000_tx_desc *tx_desc = E1000_TX_DESC(*tx_ring, i);
3437                 struct e1000_tx_buffer *buffer_info = &tx_ring->buffer_info[i];
3438                 struct my_u { __le64 a; __le64 b; };
3439                 struct my_u *u = (struct my_u *)tx_desc;
3440                 const char *type;
3441
3442                 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
3443                         type = "NTC/U";
3444                 else if (i == tx_ring->next_to_use)
3445                         type = "NTU";
3446                 else if (i == tx_ring->next_to_clean)
3447                         type = "NTC";
3448                 else
3449                         type = "";
3450
3451                 pr_info("T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X %016llX %p %s\n",
3452                         ((le64_to_cpu(u->b) & (1<<20)) ? 'd' : 'c'), i,
3453                         le64_to_cpu(u->a), le64_to_cpu(u->b),
3454                         (u64)buffer_info->dma, buffer_info->length,
3455                         buffer_info->next_to_watch,
3456                         (u64)buffer_info->time_stamp, buffer_info->skb, type);
3457         }
3458
3459 rx_ring_summary:
3460         /* receive dump */
3461         pr_info("\nRX Desc ring dump\n");
3462
3463         /* Legacy Receive Descriptor Format
3464          *
3465          * +-----------------------------------------------------+
3466          * |                Buffer Address [63:0]                |
3467          * +-----------------------------------------------------+
3468          * | VLAN Tag | Errors | Status 0 | Packet csum | Length |
3469          * +-----------------------------------------------------+
3470          * 63       48 47    40 39      32 31         16 15      0
3471          */
3472         pr_info("R[desc]      [address 63:0  ] [vl er S cks ln] [bi->dma       ] [bi->skb]\n");
3473
3474         if (!netif_msg_rx_status(adapter))
3475                 goto exit;
3476
3477         for (i = 0; rx_ring->desc && (i < rx_ring->count); i++) {
3478                 struct e1000_rx_desc *rx_desc = E1000_RX_DESC(*rx_ring, i);
3479                 struct e1000_rx_buffer *buffer_info = &rx_ring->buffer_info[i];
3480                 struct my_u { __le64 a; __le64 b; };
3481                 struct my_u *u = (struct my_u *)rx_desc;
3482                 const char *type;
3483
3484                 if (i == rx_ring->next_to_use)
3485                         type = "NTU";
3486                 else if (i == rx_ring->next_to_clean)
3487                         type = "NTC";
3488                 else
3489                         type = "";
3490
3491                 pr_info("R[0x%03X]     %016llX %016llX %016llX %p %s\n",
3492                         i, le64_to_cpu(u->a), le64_to_cpu(u->b),
3493                         (u64)buffer_info->dma, buffer_info->rxbuf.data, type);
3494         } /* for */
3495
3496         /* dump the descriptor caches */
3497         /* rx */
3498         pr_info("Rx descriptor cache in 64bit format\n");
3499         for (i = 0x6000; i <= 0x63FF ; i += 0x10) {
3500                 pr_info("R%04X: %08X|%08X %08X|%08X\n",
3501                         i,
3502                         readl(adapter->hw.hw_addr + i+4),
3503                         readl(adapter->hw.hw_addr + i),
3504                         readl(adapter->hw.hw_addr + i+12),
3505                         readl(adapter->hw.hw_addr + i+8));
3506         }
3507         /* tx */
3508         pr_info("Tx descriptor cache in 64bit format\n");
3509         for (i = 0x7000; i <= 0x73FF ; i += 0x10) {
3510                 pr_info("T%04X: %08X|%08X %08X|%08X\n",
3511                         i,
3512                         readl(adapter->hw.hw_addr + i+4),
3513                         readl(adapter->hw.hw_addr + i),
3514                         readl(adapter->hw.hw_addr + i+12),
3515                         readl(adapter->hw.hw_addr + i+8));
3516         }
3517 exit:
3518         return;
3519 }
3520
3521 /**
3522  * e1000_tx_timeout - Respond to a Tx Hang
3523  * @netdev: network interface device structure
3524  **/
3525 static void e1000_tx_timeout(struct net_device *netdev)
3526 {
3527         struct e1000_adapter *adapter = netdev_priv(netdev);
3528
3529         /* Do the reset outside of interrupt context */
3530         adapter->tx_timeout_count++;
3531         schedule_work(&adapter->reset_task);
3532 }
3533
3534 static void e1000_reset_task(struct work_struct *work)
3535 {
3536         struct e1000_adapter *adapter =
3537                 container_of(work, struct e1000_adapter, reset_task);
3538
3539         e_err(drv, "Reset adapter\n");
3540         e1000_reinit_locked(adapter);
3541 }
3542
3543 /**
3544  * e1000_change_mtu - Change the Maximum Transfer Unit
3545  * @netdev: network interface device structure
3546  * @new_mtu: new value for maximum frame size
3547  *
3548  * Returns 0 on success, negative on failure
3549  **/
3550 static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
3551 {
3552         struct e1000_adapter *adapter = netdev_priv(netdev);
3553         struct e1000_hw *hw = &adapter->hw;
3554         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3555
3556         /* Adapter-specific max frame size limits. */
3557         switch (hw->mac_type) {
3558         case e1000_undefined ... e1000_82542_rev2_1:
3559                 if (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)) {
3560                         e_err(probe, "Jumbo Frames not supported.\n");
3561                         return -EINVAL;
3562                 }
3563                 break;
3564         default:
3565                 /* Capable of supporting up to MAX_JUMBO_FRAME_SIZE limit. */
3566                 break;
3567         }
3568
3569         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
3570                 msleep(1);
3571         /* e1000_down has a dependency on max_frame_size */
3572         hw->max_frame_size = max_frame;
3573         if (netif_running(netdev)) {
3574                 /* prevent buffers from being reallocated */
3575                 adapter->alloc_rx_buf = e1000_alloc_dummy_rx_buffers;
3576                 e1000_down(adapter);
3577         }
3578
3579         /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3580          * means we reserve 2 more, this pushes us to allocate from the next
3581          * larger slab size.
3582          * i.e. RXBUFFER_2048 --> size-4096 slab
3583          * however with the new *_jumbo_rx* routines, jumbo receives will use
3584          * fragmented skbs
3585          */
3586
3587         if (max_frame <= E1000_RXBUFFER_2048)
3588                 adapter->rx_buffer_len = E1000_RXBUFFER_2048;
3589         else
3590 #if (PAGE_SIZE >= E1000_RXBUFFER_16384)
3591                 adapter->rx_buffer_len = E1000_RXBUFFER_16384;
3592 #elif (PAGE_SIZE >= E1000_RXBUFFER_4096)
3593                 adapter->rx_buffer_len = PAGE_SIZE;
3594 #endif
3595
3596         /* adjust allocation if LPE protects us, and we aren't using SBP */
3597         if (!hw->tbi_compatibility_on &&
3598             ((max_frame == (ETH_FRAME_LEN + ETH_FCS_LEN)) ||
3599              (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE)))
3600                 adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3601
3602         pr_info("%s changing MTU from %d to %d\n",
3603                 netdev->name, netdev->mtu, new_mtu);
3604         netdev->mtu = new_mtu;
3605
3606         if (netif_running(netdev))
3607                 e1000_up(adapter);
3608         else
3609                 e1000_reset(adapter);
3610
3611         clear_bit(__E1000_RESETTING, &adapter->flags);
3612
3613         return 0;
3614 }
3615
3616 /**
3617  * e1000_update_stats - Update the board statistics counters
3618  * @adapter: board private structure
3619  **/
3620 void e1000_update_stats(struct e1000_adapter *adapter)
3621 {
3622         struct net_device *netdev = adapter->netdev;
3623         struct e1000_hw *hw = &adapter->hw;
3624         struct pci_dev *pdev = adapter->pdev;
3625         unsigned long flags;
3626         u16 phy_tmp;
3627
3628 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3629
3630         /* Prevent stats update while adapter is being reset, or if the pci
3631          * connection is down.
3632          */
3633         if (adapter->link_speed == 0)
3634                 return;
3635         if (pci_channel_offline(pdev))
3636                 return;
3637
3638         spin_lock_irqsave(&adapter->stats_lock, flags);
3639
3640         /* these counters are modified from e1000_tbi_adjust_stats,
3641          * called from the interrupt context, so they must only
3642          * be written while holding adapter->stats_lock
3643          */
3644
3645         adapter->stats.crcerrs += er32(CRCERRS);
3646         adapter->stats.gprc += er32(GPRC);
3647         adapter->stats.gorcl += er32(GORCL);
3648         adapter->stats.gorch += er32(GORCH);
3649         adapter->stats.bprc += er32(BPRC);
3650         adapter->stats.mprc += er32(MPRC);
3651         adapter->stats.roc += er32(ROC);
3652
3653         adapter->stats.prc64 += er32(PRC64);
3654         adapter->stats.prc127 += er32(PRC127);
3655         adapter->stats.prc255 += er32(PRC255);
3656         adapter->stats.prc511 += er32(PRC511);
3657         adapter->stats.prc1023 += er32(PRC1023);
3658         adapter->stats.prc1522 += er32(PRC1522);
3659
3660         adapter->stats.symerrs += er32(SYMERRS);
3661         adapter->stats.mpc += er32(MPC);
3662         adapter->stats.scc += er32(SCC);
3663         adapter->stats.ecol += er32(ECOL);
3664         adapter->stats.mcc += er32(MCC);
3665         adapter->stats.latecol += er32(LATECOL);
3666         adapter->stats.dc += er32(DC);
3667         adapter->stats.sec += er32(SEC);
3668         adapter->stats.rlec += er32(RLEC);
3669         adapter->stats.xonrxc += er32(XONRXC);
3670         adapter->stats.xontxc += er32(XONTXC);
3671         adapter->stats.xoffrxc += er32(XOFFRXC);
3672         adapter->stats.xofftxc += er32(XOFFTXC);
3673         adapter->stats.fcruc += er32(FCRUC);
3674         adapter->stats.gptc += er32(GPTC);
3675         adapter->stats.gotcl += er32(GOTCL);
3676         adapter->stats.gotch += er32(GOTCH);
3677         adapter->stats.rnbc += er32(RNBC);
3678         adapter->stats.ruc += er32(RUC);
3679         adapter->stats.rfc += er32(RFC);
3680         adapter->stats.rjc += er32(RJC);
3681         adapter->stats.torl += er32(TORL);
3682         adapter->stats.torh += er32(TORH);
3683         adapter->stats.totl += er32(TOTL);
3684         adapter->stats.toth += er32(TOTH);
3685         adapter->stats.tpr += er32(TPR);
3686
3687         adapter->stats.ptc64 += er32(PTC64);
3688         adapter->stats.ptc127 += er32(PTC127);
3689         adapter->stats.ptc255 += er32(PTC255);
3690         adapter->stats.ptc511 += er32(PTC511);
3691         adapter->stats.ptc1023 += er32(PTC1023);
3692         adapter->stats.ptc1522 += er32(PTC1522);
3693
3694         adapter->stats.mptc += er32(MPTC);
3695         adapter->stats.bptc += er32(BPTC);
3696
3697         /* used for adaptive IFS */
3698
3699         hw->tx_packet_delta = er32(TPT);
3700         adapter->stats.tpt += hw->tx_packet_delta;
3701         hw->collision_delta = er32(COLC);
3702         adapter->stats.colc += hw->collision_delta;
3703
3704         if (hw->mac_type >= e1000_82543) {
3705                 adapter->stats.algnerrc += er32(ALGNERRC);
3706                 adapter->stats.rxerrc += er32(RXERRC);
3707                 adapter->stats.tncrs += er32(TNCRS);
3708                 adapter->stats.cexterr += er32(CEXTERR);
3709                 adapter->stats.tsctc += er32(TSCTC);
3710                 adapter->stats.tsctfc += er32(TSCTFC);
3711         }
3712
3713         /* Fill out the OS statistics structure */
3714         netdev->stats.multicast = adapter->stats.mprc;
3715         netdev->stats.collisions = adapter->stats.colc;
3716
3717         /* Rx Errors */
3718
3719         /* RLEC on some newer hardware can be incorrect so build
3720          * our own version based on RUC and ROC
3721          */
3722         netdev->stats.rx_errors = adapter->stats.rxerrc +
3723                 adapter->stats.crcerrs + adapter->stats.algnerrc +
3724                 adapter->stats.ruc + adapter->stats.roc +
3725                 adapter->stats.cexterr;
3726         adapter->stats.rlerrc = adapter->stats.ruc + adapter->stats.roc;
3727         netdev->stats.rx_length_errors = adapter->stats.rlerrc;
3728         netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3729         netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3730         netdev->stats.rx_missed_errors = adapter->stats.mpc;
3731
3732         /* Tx Errors */
3733         adapter->stats.txerrc = adapter->stats.ecol + adapter->stats.latecol;
3734         netdev->stats.tx_errors = adapter->stats.txerrc;
3735         netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3736         netdev->stats.tx_window_errors = adapter->stats.latecol;
3737         netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
3738         if (hw->bad_tx_carr_stats_fd &&
3739             adapter->link_duplex == FULL_DUPLEX) {
3740                 netdev->stats.tx_carrier_errors = 0;
3741                 adapter->stats.tncrs = 0;
3742         }
3743
3744         /* Tx Dropped needs to be maintained elsewhere */
3745
3746         /* Phy Stats */
3747         if (hw->media_type == e1000_media_type_copper) {
3748                 if ((adapter->link_speed == SPEED_1000) &&
3749                    (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
3750                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3751                         adapter->phy_stats.idle_errors += phy_tmp;
3752                 }
3753
3754                 if ((hw->mac_type <= e1000_82546) &&
3755                    (hw->phy_type == e1000_phy_m88) &&
3756                    !e1000_read_phy_reg(hw, M88E1000_RX_ERR_CNTR, &phy_tmp))
3757                         adapter->phy_stats.receive_errors += phy_tmp;
3758         }
3759
3760         /* Management Stats */
3761         if (hw->has_smbus) {
3762                 adapter->stats.mgptc += er32(MGTPTC);
3763                 adapter->stats.mgprc += er32(MGTPRC);
3764                 adapter->stats.mgpdc += er32(MGTPDC);
3765         }
3766
3767         spin_unlock_irqrestore(&adapter->stats_lock, flags);
3768 }
3769
3770 /**
3771  * e1000_intr - Interrupt Handler
3772  * @irq: interrupt number
3773  * @data: pointer to a network interface device structure
3774  **/
3775 static irqreturn_t e1000_intr(int irq, void *data)
3776 {
3777         struct net_device *netdev = data;
3778         struct e1000_adapter *adapter = netdev_priv(netdev);
3779         struct e1000_hw *hw = &adapter->hw;
3780         u32 icr = er32(ICR);
3781
3782         if (unlikely((!icr)))
3783                 return IRQ_NONE;  /* Not our interrupt */
3784
3785         /* we might have caused the interrupt, but the above
3786          * read cleared it, and just in case the driver is
3787          * down there is nothing to do so return handled
3788          */
3789         if (unlikely(test_bit(__E1000_DOWN, &adapter->flags)))
3790                 return IRQ_HANDLED;
3791
3792         if (unlikely(icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))) {
3793                 hw->get_link_status = 1;
3794                 /* guard against interrupt when we're going down */
3795                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3796                         schedule_delayed_work(&adapter->watchdog_task, 1);
3797         }
3798
3799         /* disable interrupts, without the synchronize_irq bit */
3800         ew32(IMC, ~0);
3801         E1000_WRITE_FLUSH();
3802
3803         if (likely(napi_schedule_prep(&adapter->napi))) {
3804                 adapter->total_tx_bytes = 0;
3805                 adapter->total_tx_packets = 0;
3806                 adapter->total_rx_bytes = 0;
3807                 adapter->total_rx_packets = 0;
3808                 __napi_schedule(&adapter->napi);
3809         } else {
3810                 /* this really should not happen! if it does it is basically a
3811                  * bug, but not a hard error, so enable ints and continue
3812                  */
3813                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3814                         e1000_irq_enable(adapter);
3815         }
3816
3817         return IRQ_HANDLED;
3818 }
3819
3820 /**
3821  * e1000_clean - NAPI Rx polling callback
3822  * @adapter: board private structure
3823  **/
3824 static int e1000_clean(struct napi_struct *napi, int budget)
3825 {
3826         struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
3827                                                      napi);
3828         int tx_clean_complete = 0, work_done = 0;
3829
3830         tx_clean_complete = e1000_clean_tx_irq(adapter, &adapter->tx_ring[0]);
3831
3832         adapter->clean_rx(adapter, &adapter->rx_ring[0], &work_done, budget);
3833
3834         if (!tx_clean_complete)
3835                 work_done = budget;
3836
3837         /* If budget not fully consumed, exit the polling mode */
3838         if (work_done < budget) {
3839                 if (likely(adapter->itr_setting & 3))
3840                         e1000_set_itr(adapter);
3841                 napi_complete_done(napi, work_done);
3842                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3843                         e1000_irq_enable(adapter);
3844         }
3845
3846         return work_done;
3847 }
3848
3849 /**
3850  * e1000_clean_tx_irq - Reclaim resources after transmit completes
3851  * @adapter: board private structure
3852  **/
3853 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
3854                                struct e1000_tx_ring *tx_ring)
3855 {
3856         struct e1000_hw *hw = &adapter->hw;
3857         struct net_device *netdev = adapter->netdev;
3858         struct e1000_tx_desc *tx_desc, *eop_desc;
3859         struct e1000_tx_buffer *buffer_info;
3860         unsigned int i, eop;
3861         unsigned int count = 0;
3862         unsigned int total_tx_bytes = 0, total_tx_packets = 0;
3863         unsigned int bytes_compl = 0, pkts_compl = 0;
3864
3865         i = tx_ring->next_to_clean;
3866         eop = tx_ring->buffer_info[i].next_to_watch;
3867         eop_desc = E1000_TX_DESC(*tx_ring, eop);
3868
3869         while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
3870                (count < tx_ring->count)) {
3871                 bool cleaned = false;
3872                 dma_rmb();      /* read buffer_info after eop_desc */
3873                 for ( ; !cleaned; count++) {
3874                         tx_desc = E1000_TX_DESC(*tx_ring, i);
3875                         buffer_info = &tx_ring->buffer_info[i];
3876                         cleaned = (i == eop);
3877
3878                         if (cleaned) {
3879                                 total_tx_packets += buffer_info->segs;
3880                                 total_tx_bytes += buffer_info->bytecount;
3881                                 if (buffer_info->skb) {
3882                                         bytes_compl += buffer_info->skb->len;
3883                                         pkts_compl++;
3884                                 }
3885
3886                         }
3887                         e1000_unmap_and_free_tx_resource(adapter, buffer_info);
3888                         tx_desc->upper.data = 0;
3889
3890                         if (unlikely(++i == tx_ring->count))
3891                                 i = 0;
3892                 }
3893
3894                 eop = tx_ring->buffer_info[i].next_to_watch;
3895                 eop_desc = E1000_TX_DESC(*tx_ring, eop);
3896         }
3897
3898         /* Synchronize with E1000_DESC_UNUSED called from e1000_xmit_frame,
3899          * which will reuse the cleaned buffers.
3900          */
3901         smp_store_release(&tx_ring->next_to_clean, i);
3902
3903         netdev_completed_queue(netdev, pkts_compl, bytes_compl);
3904
3905 #define TX_WAKE_THRESHOLD 32
3906         if (unlikely(count && netif_carrier_ok(netdev) &&
3907                      E1000_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD)) {
3908                 /* Make sure that anybody stopping the queue after this
3909                  * sees the new next_to_clean.
3910                  */
3911                 smp_mb();
3912
3913                 if (netif_queue_stopped(netdev) &&
3914                     !(test_bit(__E1000_DOWN, &adapter->flags))) {
3915                         netif_wake_queue(netdev);
3916                         ++adapter->restart_queue;
3917                 }
3918         }
3919
3920         if (adapter->detect_tx_hung) {
3921                 /* Detect a transmit hang in hardware, this serializes the
3922                  * check with the clearing of time_stamp and movement of i
3923                  */
3924                 adapter->detect_tx_hung = false;
3925                 if (tx_ring->buffer_info[eop].time_stamp &&
3926                     time_after(jiffies, tx_ring->buffer_info[eop].time_stamp +
3927                                (adapter->tx_timeout_factor * HZ)) &&
3928                     !(er32(STATUS) & E1000_STATUS_TXOFF)) {
3929
3930                         /* detected Tx unit hang */
3931                         e_err(drv, "Detected Tx Unit Hang\n"
3932                               "  Tx Queue             <%lu>\n"
3933                               "  TDH                  <%x>\n"
3934                               "  TDT                  <%x>\n"
3935                               "  next_to_use          <%x>\n"
3936                               "  next_to_clean        <%x>\n"
3937                               "buffer_info[next_to_clean]\n"
3938                               "  time_stamp           <%lx>\n"
3939                               "  next_to_watch        <%x>\n"
3940                               "  jiffies              <%lx>\n"
3941                               "  next_to_watch.status <%x>\n",
3942                                 (unsigned long)(tx_ring - adapter->tx_ring),
3943                                 readl(hw->hw_addr + tx_ring->tdh),
3944                                 readl(hw->hw_addr + tx_ring->tdt),
3945                                 tx_ring->next_to_use,
3946                                 tx_ring->next_to_clean,
3947                                 tx_ring->buffer_info[eop].time_stamp,
3948                                 eop,
3949                                 jiffies,
3950                                 eop_desc->upper.fields.status);
3951                         e1000_dump(adapter);
3952                         netif_stop_queue(netdev);
3953                 }
3954         }
3955         adapter->total_tx_bytes += total_tx_bytes;
3956         adapter->total_tx_packets += total_tx_packets;
3957         netdev->stats.tx_bytes += total_tx_bytes;
3958         netdev->stats.tx_packets += total_tx_packets;
3959         return count < tx_ring->count;
3960 }
3961
3962 /**
3963  * e1000_rx_checksum - Receive Checksum Offload for 82543
3964  * @adapter:     board private structure
3965  * @status_err:  receive descriptor status and error fields
3966  * @csum:        receive descriptor csum field
3967  * @sk_buff:     socket buffer with received data
3968  **/
3969 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
3970                               u32 csum, struct sk_buff *skb)
3971 {
3972         struct e1000_hw *hw = &adapter->hw;
3973         u16 status = (u16)status_err;
3974         u8 errors = (u8)(status_err >> 24);
3975
3976         skb_checksum_none_assert(skb);
3977
3978         /* 82543 or newer only */
3979         if (unlikely(hw->mac_type < e1000_82543))
3980                 return;
3981         /* Ignore Checksum bit is set */
3982         if (unlikely(status & E1000_RXD_STAT_IXSM))
3983                 return;
3984         /* TCP/UDP checksum error bit is set */
3985         if (unlikely(errors & E1000_RXD_ERR_TCPE)) {
3986                 /* let the stack verify checksum errors */
3987                 adapter->hw_csum_err++;
3988                 return;
3989         }
3990         /* TCP/UDP Checksum has not been calculated */
3991         if (!(status & E1000_RXD_STAT_TCPCS))
3992                 return;
3993
3994         /* It must be a TCP or UDP packet with a valid checksum */
3995         if (likely(status & E1000_RXD_STAT_TCPCS)) {
3996                 /* TCP checksum is good */
3997                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3998         }
3999         adapter->hw_csum_good++;
4000 }
4001
4002 /**
4003  * e1000_consume_page - helper function for jumbo Rx path
4004  **/
4005 static void e1000_consume_page(struct e1000_rx_buffer *bi, struct sk_buff *skb,
4006                                u16 length)
4007 {
4008         bi->rxbuf.page = NULL;
4009         skb->len += length;
4010         skb->data_len += length;
4011         skb->truesize += PAGE_SIZE;
4012 }
4013
4014 /**
4015  * e1000_receive_skb - helper function to handle rx indications
4016  * @adapter: board private structure
4017  * @status: descriptor status field as written by hardware
4018  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
4019  * @skb: pointer to sk_buff to be indicated to stack
4020  */
4021 static void e1000_receive_skb(struct e1000_adapter *adapter, u8 status,
4022                               __le16 vlan, struct sk_buff *skb)
4023 {
4024         skb->protocol = eth_type_trans(skb, adapter->netdev);
4025
4026         if (status & E1000_RXD_STAT_VP) {
4027                 u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4028
4029                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4030         }
4031         napi_gro_receive(&adapter->napi, skb);
4032 }
4033
4034 /**
4035  * e1000_tbi_adjust_stats
4036  * @hw: Struct containing variables accessed by shared code
4037  * @frame_len: The length of the frame in question
4038  * @mac_addr: The Ethernet destination address of the frame in question
4039  *
4040  * Adjusts the statistic counters when a frame is accepted by TBI_ACCEPT
4041  */
4042 static void e1000_tbi_adjust_stats(struct e1000_hw *hw,
4043                                    struct e1000_hw_stats *stats,
4044                                    u32 frame_len, const u8 *mac_addr)
4045 {
4046         u64 carry_bit;
4047
4048         /* First adjust the frame length. */
4049         frame_len--;
4050         /* We need to adjust the statistics counters, since the hardware
4051          * counters overcount this packet as a CRC error and undercount
4052          * the packet as a good packet
4053          */
4054         /* This packet should not be counted as a CRC error. */
4055         stats->crcerrs--;
4056         /* This packet does count as a Good Packet Received. */
4057         stats->gprc++;
4058
4059         /* Adjust the Good Octets received counters */
4060         carry_bit = 0x80000000 & stats->gorcl;
4061         stats->gorcl += frame_len;
4062         /* If the high bit of Gorcl (the low 32 bits of the Good Octets
4063          * Received Count) was one before the addition,
4064          * AND it is zero after, then we lost the carry out,
4065          * need to add one to Gorch (Good Octets Received Count High).
4066          * This could be simplified if all environments supported
4067          * 64-bit integers.
4068          */
4069         if (carry_bit && ((stats->gorcl & 0x80000000) == 0))
4070                 stats->gorch++;
4071         /* Is this a broadcast or multicast?  Check broadcast first,
4072          * since the test for a multicast frame will test positive on
4073          * a broadcast frame.
4074          */
4075         if (is_broadcast_ether_addr(mac_addr))
4076                 stats->bprc++;
4077         else if (is_multicast_ether_addr(mac_addr))
4078                 stats->mprc++;
4079
4080         if (frame_len == hw->max_frame_size) {
4081                 /* In this case, the hardware has overcounted the number of
4082                  * oversize frames.
4083                  */
4084                 if (stats->roc > 0)
4085                         stats->roc--;
4086         }
4087
4088         /* Adjust the bin counters when the extra byte put the frame in the
4089          * wrong bin. Remember that the frame_len was adjusted above.
4090          */
4091         if (frame_len == 64) {
4092                 stats->prc64++;
4093                 stats->prc127--;
4094         } else if (frame_len == 127) {
4095                 stats->prc127++;
4096                 stats->prc255--;
4097         } else if (frame_len == 255) {
4098                 stats->prc255++;
4099                 stats->prc511--;
4100         } else if (frame_len == 511) {
4101                 stats->prc511++;
4102                 stats->prc1023--;
4103         } else if (frame_len == 1023) {
4104                 stats->prc1023++;
4105                 stats->prc1522--;
4106         } else if (frame_len == 1522) {
4107                 stats->prc1522++;
4108         }
4109 }
4110
4111 static bool e1000_tbi_should_accept(struct e1000_adapter *adapter,
4112                                     u8 status, u8 errors,
4113                                     u32 length, const u8 *data)
4114 {
4115         struct e1000_hw *hw = &adapter->hw;
4116         u8 last_byte = *(data + length - 1);
4117
4118         if (TBI_ACCEPT(hw, status, errors, length, last_byte)) {
4119                 unsigned long irq_flags;
4120
4121                 spin_lock_irqsave(&adapter->stats_lock, irq_flags);
4122                 e1000_tbi_adjust_stats(hw, &adapter->stats, length, data);
4123                 spin_unlock_irqrestore(&adapter->stats_lock, irq_flags);
4124
4125                 return true;
4126         }
4127
4128         return false;
4129 }
4130
4131 static struct sk_buff *e1000_alloc_rx_skb(struct e1000_adapter *adapter,
4132                                           unsigned int bufsz)
4133 {
4134         struct sk_buff *skb = napi_alloc_skb(&adapter->napi, bufsz);
4135
4136         if (unlikely(!skb))
4137                 adapter->alloc_rx_buff_failed++;
4138         return skb;
4139 }
4140
4141 /**
4142  * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
4143  * @adapter: board private structure
4144  * @rx_ring: ring to clean
4145  * @work_done: amount of napi work completed this call
4146  * @work_to_do: max amount of work allowed for this call to do
4147  *
4148  * the return value indicates whether actual cleaning was done, there
4149  * is no guarantee that everything was cleaned
4150  */
4151 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
4152                                      struct e1000_rx_ring *rx_ring,
4153                                      int *work_done, int work_to_do)
4154 {
4155         struct net_device *netdev = adapter->netdev;
4156         struct pci_dev *pdev = adapter->pdev;
4157         struct e1000_rx_desc *rx_desc, *next_rxd;
4158         struct e1000_rx_buffer *buffer_info, *next_buffer;
4159         u32 length;
4160         unsigned int i;
4161         int cleaned_count = 0;
4162         bool cleaned = false;
4163         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4164
4165         i = rx_ring->next_to_clean;
4166         rx_desc = E1000_RX_DESC(*rx_ring, i);
4167         buffer_info = &rx_ring->buffer_info[i];
4168
4169         while (rx_desc->status & E1000_RXD_STAT_DD) {
4170                 struct sk_buff *skb;
4171                 u8 status;
4172
4173                 if (*work_done >= work_to_do)
4174                         break;
4175                 (*work_done)++;
4176                 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4177
4178                 status = rx_desc->status;
4179
4180                 if (++i == rx_ring->count)
4181                         i = 0;
4182
4183                 next_rxd = E1000_RX_DESC(*rx_ring, i);
4184                 prefetch(next_rxd);
4185
4186                 next_buffer = &rx_ring->buffer_info[i];
4187
4188                 cleaned = true;
4189                 cleaned_count++;
4190                 dma_unmap_page(&pdev->dev, buffer_info->dma,
4191                                adapter->rx_buffer_len, DMA_FROM_DEVICE);
4192                 buffer_info->dma = 0;
4193
4194                 length = le16_to_cpu(rx_desc->length);
4195
4196                 /* errors is only valid for DD + EOP descriptors */
4197                 if (unlikely((status & E1000_RXD_STAT_EOP) &&
4198                     (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK))) {
4199                         u8 *mapped = page_address(buffer_info->rxbuf.page);
4200
4201                         if (e1000_tbi_should_accept(adapter, status,
4202                                                     rx_desc->errors,
4203                                                     length, mapped)) {
4204                                 length--;
4205                         } else if (netdev->features & NETIF_F_RXALL) {
4206                                 goto process_skb;
4207                         } else {
4208                                 /* an error means any chain goes out the window
4209                                  * too
4210                                  */
4211                                 if (rx_ring->rx_skb_top)
4212                                         dev_kfree_skb(rx_ring->rx_skb_top);
4213                                 rx_ring->rx_skb_top = NULL;
4214                                 goto next_desc;
4215                         }
4216                 }
4217
4218 #define rxtop rx_ring->rx_skb_top
4219 process_skb:
4220                 if (!(status & E1000_RXD_STAT_EOP)) {
4221                         /* this descriptor is only the beginning (or middle) */
4222                         if (!rxtop) {
4223                                 /* this is the beginning of a chain */
4224                                 rxtop = napi_get_frags(&adapter->napi);
4225                                 if (!rxtop)
4226                                         break;
4227
4228                                 skb_fill_page_desc(rxtop, 0,
4229                                                    buffer_info->rxbuf.page,
4230                                                    0, length);
4231                         } else {
4232                                 /* this is the middle of a chain */
4233                                 skb_fill_page_desc(rxtop,
4234                                     skb_shinfo(rxtop)->nr_frags,
4235                                     buffer_info->rxbuf.page, 0, length);
4236                         }
4237                         e1000_consume_page(buffer_info, rxtop, length);
4238                         goto next_desc;
4239                 } else {
4240                         if (rxtop) {
4241                                 /* end of the chain */
4242                                 skb_fill_page_desc(rxtop,
4243                                     skb_shinfo(rxtop)->nr_frags,
4244                                     buffer_info->rxbuf.page, 0, length);
4245                                 skb = rxtop;
4246                                 rxtop = NULL;
4247                                 e1000_consume_page(buffer_info, skb, length);
4248                         } else {
4249                                 struct page *p;
4250                                 /* no chain, got EOP, this buf is the packet
4251                                  * copybreak to save the put_page/alloc_page
4252                                  */
4253                                 p = buffer_info->rxbuf.page;
4254                                 if (length <= copybreak) {
4255                                         u8 *vaddr;
4256
4257                                         if (likely(!(netdev->features & NETIF_F_RXFCS)))
4258                                                 length -= 4;
4259                                         skb = e1000_alloc_rx_skb(adapter,
4260                                                                  length);
4261                                         if (!skb)
4262                                                 break;
4263
4264                                         vaddr = kmap_atomic(p);
4265                                         memcpy(skb_tail_pointer(skb), vaddr,
4266                                                length);
4267                                         kunmap_atomic(vaddr);
4268                                         /* re-use the page, so don't erase
4269                                          * buffer_info->rxbuf.page
4270                                          */
4271                                         skb_put(skb, length);
4272                                         e1000_rx_checksum(adapter,
4273                                                           status | rx_desc->errors << 24,
4274                                                           le16_to_cpu(rx_desc->csum), skb);
4275
4276                                         total_rx_bytes += skb->len;
4277                                         total_rx_packets++;
4278
4279                                         e1000_receive_skb(adapter, status,
4280                                                           rx_desc->special, skb);
4281                                         goto next_desc;
4282                                 } else {
4283                                         skb = napi_get_frags(&adapter->napi);
4284                                         if (!skb) {
4285                                                 adapter->alloc_rx_buff_failed++;
4286                                                 break;
4287                                         }
4288                                         skb_fill_page_desc(skb, 0, p, 0,
4289                                                            length);
4290                                         e1000_consume_page(buffer_info, skb,
4291                                                            length);
4292                                 }
4293                         }
4294                 }
4295
4296                 /* Receive Checksum Offload XXX recompute due to CRC strip? */
4297                 e1000_rx_checksum(adapter,
4298                                   (u32)(status) |
4299                                   ((u32)(rx_desc->errors) << 24),
4300                                   le16_to_cpu(rx_desc->csum), skb);
4301
4302                 total_rx_bytes += (skb->len - 4); /* don't count FCS */
4303                 if (likely(!(netdev->features & NETIF_F_RXFCS)))
4304                         pskb_trim(skb, skb->len - 4);
4305                 total_rx_packets++;
4306
4307                 if (status & E1000_RXD_STAT_VP) {
4308                         __le16 vlan = rx_desc->special;
4309                         u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4310
4311                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4312                 }
4313
4314                 napi_gro_frags(&adapter->napi);
4315
4316 next_desc:
4317                 rx_desc->status = 0;
4318
4319                 /* return some buffers to hardware, one at a time is too slow */
4320                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4321                         adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4322                         cleaned_count = 0;
4323                 }
4324
4325                 /* use prefetched values */
4326                 rx_desc = next_rxd;
4327                 buffer_info = next_buffer;
4328         }
4329         rx_ring->next_to_clean = i;
4330
4331         cleaned_count = E1000_DESC_UNUSED(rx_ring);
4332         if (cleaned_count)
4333                 adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4334
4335         adapter->total_rx_packets += total_rx_packets;
4336         adapter->total_rx_bytes += total_rx_bytes;
4337         netdev->stats.rx_bytes += total_rx_bytes;
4338         netdev->stats.rx_packets += total_rx_packets;
4339         return cleaned;
4340 }
4341
4342 /* this should improve performance for small packets with large amounts
4343  * of reassembly being done in the stack
4344  */
4345 static struct sk_buff *e1000_copybreak(struct e1000_adapter *adapter,
4346                                        struct e1000_rx_buffer *buffer_info,
4347                                        u32 length, const void *data)
4348 {
4349         struct sk_buff *skb;
4350
4351         if (length > copybreak)
4352                 return NULL;
4353
4354         skb = e1000_alloc_rx_skb(adapter, length);
4355         if (!skb)
4356                 return NULL;
4357
4358         dma_sync_single_for_cpu(&adapter->pdev->dev, buffer_info->dma,
4359                                 length, DMA_FROM_DEVICE);
4360
4361         skb_put_data(skb, data, length);
4362
4363         return skb;
4364 }
4365
4366 /**
4367  * e1000_clean_rx_irq - Send received data up the network stack; legacy
4368  * @adapter: board private structure
4369  * @rx_ring: ring to clean
4370  * @work_done: amount of napi work completed this call
4371  * @work_to_do: max amount of work allowed for this call to do
4372  */
4373 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
4374                                struct e1000_rx_ring *rx_ring,
4375                                int *work_done, int work_to_do)
4376 {
4377         struct net_device *netdev = adapter->netdev;
4378         struct pci_dev *pdev = adapter->pdev;
4379         struct e1000_rx_desc *rx_desc, *next_rxd;
4380         struct e1000_rx_buffer *buffer_info, *next_buffer;
4381         u32 length;
4382         unsigned int i;
4383         int cleaned_count = 0;
4384         bool cleaned = false;
4385         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4386
4387         i = rx_ring->next_to_clean;
4388         rx_desc = E1000_RX_DESC(*rx_ring, i);
4389         buffer_info = &rx_ring->buffer_info[i];
4390
4391         while (rx_desc->status & E1000_RXD_STAT_DD) {
4392                 struct sk_buff *skb;
4393                 u8 *data;
4394                 u8 status;
4395
4396                 if (*work_done >= work_to_do)
4397                         break;
4398                 (*work_done)++;
4399                 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4400
4401                 status = rx_desc->status;
4402                 length = le16_to_cpu(rx_desc->length);
4403
4404                 data = buffer_info->rxbuf.data;
4405                 prefetch(data);
4406                 skb = e1000_copybreak(adapter, buffer_info, length, data);
4407                 if (!skb) {
4408                         unsigned int frag_len = e1000_frag_len(adapter);
4409
4410                         skb = build_skb(data - E1000_HEADROOM, frag_len);
4411                         if (!skb) {
4412                                 adapter->alloc_rx_buff_failed++;
4413                                 break;
4414                         }
4415
4416                         skb_reserve(skb, E1000_HEADROOM);
4417                         dma_unmap_single(&pdev->dev, buffer_info->dma,
4418                                          adapter->rx_buffer_len,
4419                                          DMA_FROM_DEVICE);
4420                         buffer_info->dma = 0;
4421                         buffer_info->rxbuf.data = NULL;
4422                 }
4423
4424                 if (++i == rx_ring->count)
4425                         i = 0;
4426
4427                 next_rxd = E1000_RX_DESC(*rx_ring, i);
4428                 prefetch(next_rxd);
4429
4430                 next_buffer = &rx_ring->buffer_info[i];
4431
4432                 cleaned = true;
4433                 cleaned_count++;
4434
4435                 /* !EOP means multiple descriptors were used to store a single
4436                  * packet, if thats the case we need to toss it.  In fact, we
4437                  * to toss every packet with the EOP bit clear and the next
4438                  * frame that _does_ have the EOP bit set, as it is by
4439                  * definition only a frame fragment
4440                  */
4441                 if (unlikely(!(status & E1000_RXD_STAT_EOP)))
4442                         adapter->discarding = true;
4443
4444                 if (adapter->discarding) {
4445                         /* All receives must fit into a single buffer */
4446                         netdev_dbg(netdev, "Receive packet consumed multiple buffers\n");
4447                         dev_kfree_skb(skb);
4448                         if (status & E1000_RXD_STAT_EOP)
4449                                 adapter->discarding = false;
4450                         goto next_desc;
4451                 }
4452
4453                 if (unlikely(rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK)) {
4454                         if (e1000_tbi_should_accept(adapter, status,
4455                                                     rx_desc->errors,
4456                                                     length, data)) {
4457                                 length--;
4458                         } else if (netdev->features & NETIF_F_RXALL) {
4459                                 goto process_skb;
4460                         } else {
4461                                 dev_kfree_skb(skb);
4462                                 goto next_desc;
4463                         }
4464                 }
4465
4466 process_skb:
4467                 total_rx_bytes += (length - 4); /* don't count FCS */
4468                 total_rx_packets++;
4469
4470                 if (likely(!(netdev->features & NETIF_F_RXFCS)))
4471                         /* adjust length to remove Ethernet CRC, this must be
4472                          * done after the TBI_ACCEPT workaround above
4473                          */
4474                         length -= 4;
4475
4476                 if (buffer_info->rxbuf.data == NULL)
4477                         skb_put(skb, length);
4478                 else /* copybreak skb */
4479                         skb_trim(skb, length);
4480
4481                 /* Receive Checksum Offload */
4482                 e1000_rx_checksum(adapter,
4483                                   (u32)(status) |
4484                                   ((u32)(rx_desc->errors) << 24),
4485                                   le16_to_cpu(rx_desc->csum), skb);
4486
4487                 e1000_receive_skb(adapter, status, rx_desc->special, skb);
4488
4489 next_desc:
4490                 rx_desc->status = 0;
4491
4492                 /* return some buffers to hardware, one at a time is too slow */
4493                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4494                         adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4495                         cleaned_count = 0;
4496                 }
4497
4498                 /* use prefetched values */
4499                 rx_desc = next_rxd;
4500                 buffer_info = next_buffer;
4501         }
4502         rx_ring->next_to_clean = i;
4503
4504         cleaned_count = E1000_DESC_UNUSED(rx_ring);
4505         if (cleaned_count)
4506                 adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4507
4508         adapter->total_rx_packets += total_rx_packets;
4509         adapter->total_rx_bytes += total_rx_bytes;
4510         netdev->stats.rx_bytes += total_rx_bytes;
4511         netdev->stats.rx_packets += total_rx_packets;
4512         return cleaned;
4513 }
4514
4515 /**
4516  * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
4517  * @adapter: address of board private structure
4518  * @rx_ring: pointer to receive ring structure
4519  * @cleaned_count: number of buffers to allocate this pass
4520  **/
4521 static void
4522 e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
4523                              struct e1000_rx_ring *rx_ring, int cleaned_count)
4524 {
4525         struct pci_dev *pdev = adapter->pdev;
4526         struct e1000_rx_desc *rx_desc;
4527         struct e1000_rx_buffer *buffer_info;
4528         unsigned int i;
4529
4530         i = rx_ring->next_to_use;
4531         buffer_info = &rx_ring->buffer_info[i];
4532
4533         while (cleaned_count--) {
4534                 /* allocate a new page if necessary */
4535                 if (!buffer_info->rxbuf.page) {
4536                         buffer_info->rxbuf.page = alloc_page(GFP_ATOMIC);
4537                         if (unlikely(!buffer_info->rxbuf.page)) {
4538                                 adapter->alloc_rx_buff_failed++;
4539                                 break;
4540                         }
4541                 }
4542
4543                 if (!buffer_info->dma) {
4544                         buffer_info->dma = dma_map_page(&pdev->dev,
4545                                                         buffer_info->rxbuf.page, 0,
4546                                                         adapter->rx_buffer_len,
4547                                                         DMA_FROM_DEVICE);
4548                         if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4549                                 put_page(buffer_info->rxbuf.page);
4550                                 buffer_info->rxbuf.page = NULL;
4551                                 buffer_info->dma = 0;
4552                                 adapter->alloc_rx_buff_failed++;
4553                                 break;
4554                         }
4555                 }
4556
4557                 rx_desc = E1000_RX_DESC(*rx_ring, i);
4558                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4559
4560                 if (unlikely(++i == rx_ring->count))
4561                         i = 0;
4562                 buffer_info = &rx_ring->buffer_info[i];
4563         }
4564
4565         if (likely(rx_ring->next_to_use != i)) {
4566                 rx_ring->next_to_use = i;
4567                 if (unlikely(i-- == 0))
4568                         i = (rx_ring->count - 1);
4569
4570                 /* Force memory writes to complete before letting h/w
4571                  * know there are new descriptors to fetch.  (Only
4572                  * applicable for weak-ordered memory model archs,
4573                  * such as IA-64).
4574                  */
4575                 wmb();
4576                 writel(i, adapter->hw.hw_addr + rx_ring->rdt);
4577         }
4578 }
4579
4580 /**
4581  * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended
4582  * @adapter: address of board private structure
4583  **/
4584 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
4585                                    struct e1000_rx_ring *rx_ring,
4586                                    int cleaned_count)
4587 {
4588         struct e1000_hw *hw = &adapter->hw;
4589         struct pci_dev *pdev = adapter->pdev;
4590         struct e1000_rx_desc *rx_desc;
4591         struct e1000_rx_buffer *buffer_info;
4592         unsigned int i;
4593         unsigned int bufsz = adapter->rx_buffer_len;
4594
4595         i = rx_ring->next_to_use;
4596         buffer_info = &rx_ring->buffer_info[i];
4597
4598         while (cleaned_count--) {
4599                 void *data;
4600
4601                 if (buffer_info->rxbuf.data)
4602                         goto skip;
4603
4604                 data = e1000_alloc_frag(adapter);
4605                 if (!data) {
4606                         /* Better luck next round */
4607                         adapter->alloc_rx_buff_failed++;
4608                         break;
4609                 }
4610
4611                 /* Fix for errata 23, can't cross 64kB boundary */
4612                 if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4613                         void *olddata = data;
4614                         e_err(rx_err, "skb align check failed: %u bytes at "
4615                               "%p\n", bufsz, data);
4616                         /* Try again, without freeing the previous */
4617                         data = e1000_alloc_frag(adapter);
4618                         /* Failed allocation, critical failure */
4619                         if (!data) {
4620                                 skb_free_frag(olddata);
4621                                 adapter->alloc_rx_buff_failed++;
4622                                 break;
4623                         }
4624
4625                         if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4626                                 /* give up */
4627                                 skb_free_frag(data);
4628                                 skb_free_frag(olddata);
4629                                 adapter->alloc_rx_buff_failed++;
4630                                 break;
4631                         }
4632
4633                         /* Use new allocation */
4634                         skb_free_frag(olddata);
4635                 }
4636                 buffer_info->dma = dma_map_single(&pdev->dev,
4637                                                   data,
4638                                                   adapter->rx_buffer_len,
4639                                                   DMA_FROM_DEVICE);
4640                 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4641                         skb_free_frag(data);
4642                         buffer_info->dma = 0;
4643                         adapter->alloc_rx_buff_failed++;
4644                         break;
4645                 }
4646
4647                 /* XXX if it was allocated cleanly it will never map to a
4648                  * boundary crossing
4649                  */
4650
4651                 /* Fix for errata 23, can't cross 64kB boundary */
4652                 if (!e1000_check_64k_bound(adapter,
4653                                         (void *)(unsigned long)buffer_info->dma,
4654                                         adapter->rx_buffer_len)) {
4655                         e_err(rx_err, "dma align check failed: %u bytes at "
4656                               "%p\n", adapter->rx_buffer_len,
4657                               (void *)(unsigned long)buffer_info->dma);
4658
4659                         dma_unmap_single(&pdev->dev, buffer_info->dma,
4660                                          adapter->rx_buffer_len,
4661                                          DMA_FROM_DEVICE);
4662
4663                         skb_free_frag(data);
4664                         buffer_info->rxbuf.data = NULL;
4665                         buffer_info->dma = 0;
4666
4667                         adapter->alloc_rx_buff_failed++;
4668                         break;
4669                 }
4670                 buffer_info->rxbuf.data = data;
4671  skip:
4672                 rx_desc = E1000_RX_DESC(*rx_ring, i);
4673                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4674
4675                 if (unlikely(++i == rx_ring->count))
4676                         i = 0;
4677                 buffer_info = &rx_ring->buffer_info[i];
4678         }
4679
4680         if (likely(rx_ring->next_to_use != i)) {
4681                 rx_ring->next_to_use = i;
4682                 if (unlikely(i-- == 0))
4683                         i = (rx_ring->count - 1);
4684
4685                 /* Force memory writes to complete before letting h/w
4686                  * know there are new descriptors to fetch.  (Only
4687                  * applicable for weak-ordered memory model archs,
4688                  * such as IA-64).
4689                  */
4690                 wmb();
4691                 writel(i, hw->hw_addr + rx_ring->rdt);
4692         }
4693 }
4694
4695 /**
4696  * e1000_smartspeed - Workaround for SmartSpeed on 82541 and 82547 controllers.
4697  * @adapter:
4698  **/
4699 static void e1000_smartspeed(struct e1000_adapter *adapter)
4700 {
4701         struct e1000_hw *hw = &adapter->hw;
4702         u16 phy_status;
4703         u16 phy_ctrl;
4704
4705         if ((hw->phy_type != e1000_phy_igp) || !hw->autoneg ||
4706            !(hw->autoneg_advertised & ADVERTISE_1000_FULL))
4707                 return;
4708
4709         if (adapter->smartspeed == 0) {
4710                 /* If Master/Slave config fault is asserted twice,
4711                  * we assume back-to-back
4712                  */
4713                 e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4714                 if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4715                         return;
4716                 e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4717                 if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4718                         return;
4719                 e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4720                 if (phy_ctrl & CR_1000T_MS_ENABLE) {
4721                         phy_ctrl &= ~CR_1000T_MS_ENABLE;
4722                         e1000_write_phy_reg(hw, PHY_1000T_CTRL,
4723                                             phy_ctrl);
4724                         adapter->smartspeed++;
4725                         if (!e1000_phy_setup_autoneg(hw) &&
4726                            !e1000_read_phy_reg(hw, PHY_CTRL,
4727                                                &phy_ctrl)) {
4728                                 phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4729                                              MII_CR_RESTART_AUTO_NEG);
4730                                 e1000_write_phy_reg(hw, PHY_CTRL,
4731                                                     phy_ctrl);
4732                         }
4733                 }
4734                 return;
4735         } else if (adapter->smartspeed == E1000_SMARTSPEED_DOWNSHIFT) {
4736                 /* If still no link, perhaps using 2/3 pair cable */
4737                 e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4738                 phy_ctrl |= CR_1000T_MS_ENABLE;
4739                 e1000_write_phy_reg(hw, PHY_1000T_CTRL, phy_ctrl);
4740                 if (!e1000_phy_setup_autoneg(hw) &&
4741                    !e1000_read_phy_reg(hw, PHY_CTRL, &phy_ctrl)) {
4742                         phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4743                                      MII_CR_RESTART_AUTO_NEG);
4744                         e1000_write_phy_reg(hw, PHY_CTRL, phy_ctrl);
4745                 }
4746         }
4747         /* Restart process after E1000_SMARTSPEED_MAX iterations */
4748         if (adapter->smartspeed++ == E1000_SMARTSPEED_MAX)
4749                 adapter->smartspeed = 0;
4750 }
4751
4752 /**
4753  * e1000_ioctl -
4754  * @netdev:
4755  * @ifreq:
4756  * @cmd:
4757  **/
4758 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
4759 {
4760         switch (cmd) {
4761         case SIOCGMIIPHY:
4762         case SIOCGMIIREG:
4763         case SIOCSMIIREG:
4764                 return e1000_mii_ioctl(netdev, ifr, cmd);
4765         default:
4766                 return -EOPNOTSUPP;
4767         }
4768 }
4769
4770 /**
4771  * e1000_mii_ioctl -
4772  * @netdev:
4773  * @ifreq:
4774  * @cmd:
4775  **/
4776 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
4777                            int cmd)
4778 {
4779         struct e1000_adapter *adapter = netdev_priv(netdev);
4780         struct e1000_hw *hw = &adapter->hw;
4781         struct mii_ioctl_data *data = if_mii(ifr);
4782         int retval;
4783         u16 mii_reg;
4784         unsigned long flags;
4785
4786         if (hw->media_type != e1000_media_type_copper)
4787                 return -EOPNOTSUPP;
4788
4789         switch (cmd) {
4790         case SIOCGMIIPHY:
4791                 data->phy_id = hw->phy_addr;
4792                 break;
4793         case SIOCGMIIREG:
4794                 spin_lock_irqsave(&adapter->stats_lock, flags);
4795                 if (e1000_read_phy_reg(hw, data->reg_num & 0x1F,
4796                                    &data->val_out)) {
4797                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4798                         return -EIO;
4799                 }
4800                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4801                 break;
4802         case SIOCSMIIREG:
4803                 if (data->reg_num & ~(0x1F))
4804                         return -EFAULT;
4805                 mii_reg = data->val_in;
4806                 spin_lock_irqsave(&adapter->stats_lock, flags);
4807                 if (e1000_write_phy_reg(hw, data->reg_num,
4808                                         mii_reg)) {
4809                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4810                         return -EIO;
4811                 }
4812                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4813                 if (hw->media_type == e1000_media_type_copper) {
4814                         switch (data->reg_num) {
4815                         case PHY_CTRL:
4816                                 if (mii_reg & MII_CR_POWER_DOWN)
4817                                         break;
4818                                 if (mii_reg & MII_CR_AUTO_NEG_EN) {
4819                                         hw->autoneg = 1;
4820                                         hw->autoneg_advertised = 0x2F;
4821                                 } else {
4822                                         u32 speed;
4823                                         if (mii_reg & 0x40)
4824                                                 speed = SPEED_1000;
4825                                         else if (mii_reg & 0x2000)
4826                                                 speed = SPEED_100;
4827                                         else
4828                                                 speed = SPEED_10;
4829                                         retval = e1000_set_spd_dplx(
4830                                                 adapter, speed,
4831                                                 ((mii_reg & 0x100)
4832                                                  ? DUPLEX_FULL :
4833                                                  DUPLEX_HALF));
4834                                         if (retval)
4835                                                 return retval;
4836                                 }
4837                                 if (netif_running(adapter->netdev))
4838                                         e1000_reinit_locked(adapter);
4839                                 else
4840                                         e1000_reset(adapter);
4841                                 break;
4842                         case M88E1000_PHY_SPEC_CTRL:
4843                         case M88E1000_EXT_PHY_SPEC_CTRL:
4844                                 if (e1000_phy_reset(hw))
4845                                         return -EIO;
4846                                 break;
4847                         }
4848                 } else {
4849                         switch (data->reg_num) {
4850                         case PHY_CTRL:
4851                                 if (mii_reg & MII_CR_POWER_DOWN)
4852                                         break;
4853                                 if (netif_running(adapter->netdev))
4854                                         e1000_reinit_locked(adapter);
4855                                 else
4856                                         e1000_reset(adapter);
4857                                 break;
4858                         }
4859                 }
4860                 break;
4861         default:
4862                 return -EOPNOTSUPP;
4863         }
4864         return E1000_SUCCESS;
4865 }
4866
4867 void e1000_pci_set_mwi(struct e1000_hw *hw)
4868 {
4869         struct e1000_adapter *adapter = hw->back;
4870         int ret_val = pci_set_mwi(adapter->pdev);
4871
4872         if (ret_val)
4873                 e_err(probe, "Error in setting MWI\n");
4874 }
4875
4876 void e1000_pci_clear_mwi(struct e1000_hw *hw)
4877 {
4878         struct e1000_adapter *adapter = hw->back;
4879
4880         pci_clear_mwi(adapter->pdev);
4881 }
4882
4883 int e1000_pcix_get_mmrbc(struct e1000_hw *hw)
4884 {
4885         struct e1000_adapter *adapter = hw->back;
4886         return pcix_get_mmrbc(adapter->pdev);
4887 }
4888
4889 void e1000_pcix_set_mmrbc(struct e1000_hw *hw, int mmrbc)
4890 {
4891         struct e1000_adapter *adapter = hw->back;
4892         pcix_set_mmrbc(adapter->pdev, mmrbc);
4893 }
4894
4895 void e1000_io_write(struct e1000_hw *hw, unsigned long port, u32 value)
4896 {
4897         outl(value, port);
4898 }
4899
4900 static bool e1000_vlan_used(struct e1000_adapter *adapter)
4901 {
4902         u16 vid;
4903
4904         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
4905                 return true;
4906         return false;
4907 }
4908
4909 static void __e1000_vlan_mode(struct e1000_adapter *adapter,
4910                               netdev_features_t features)
4911 {
4912         struct e1000_hw *hw = &adapter->hw;
4913         u32 ctrl;
4914
4915         ctrl = er32(CTRL);
4916         if (features & NETIF_F_HW_VLAN_CTAG_RX) {
4917                 /* enable VLAN tag insert/strip */
4918                 ctrl |= E1000_CTRL_VME;
4919         } else {
4920                 /* disable VLAN tag insert/strip */
4921                 ctrl &= ~E1000_CTRL_VME;
4922         }
4923         ew32(CTRL, ctrl);
4924 }
4925 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
4926                                      bool filter_on)
4927 {
4928         struct e1000_hw *hw = &adapter->hw;
4929         u32 rctl;
4930
4931         if (!test_bit(__E1000_DOWN, &adapter->flags))
4932                 e1000_irq_disable(adapter);
4933
4934         __e1000_vlan_mode(adapter, adapter->netdev->features);
4935         if (filter_on) {
4936                 /* enable VLAN receive filtering */
4937                 rctl = er32(RCTL);
4938                 rctl &= ~E1000_RCTL_CFIEN;
4939                 if (!(adapter->netdev->flags & IFF_PROMISC))
4940                         rctl |= E1000_RCTL_VFE;
4941                 ew32(RCTL, rctl);
4942                 e1000_update_mng_vlan(adapter);
4943         } else {
4944                 /* disable VLAN receive filtering */
4945                 rctl = er32(RCTL);
4946                 rctl &= ~E1000_RCTL_VFE;
4947                 ew32(RCTL, rctl);
4948         }
4949
4950         if (!test_bit(__E1000_DOWN, &adapter->flags))
4951                 e1000_irq_enable(adapter);
4952 }
4953
4954 static void e1000_vlan_mode(struct net_device *netdev,
4955                             netdev_features_t features)
4956 {
4957         struct e1000_adapter *adapter = netdev_priv(netdev);
4958
4959         if (!test_bit(__E1000_DOWN, &adapter->flags))
4960                 e1000_irq_disable(adapter);
4961
4962         __e1000_vlan_mode(adapter, features);
4963
4964         if (!test_bit(__E1000_DOWN, &adapter->flags))
4965                 e1000_irq_enable(adapter);
4966 }
4967
4968 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
4969                                  __be16 proto, u16 vid)
4970 {
4971         struct e1000_adapter *adapter = netdev_priv(netdev);
4972         struct e1000_hw *hw = &adapter->hw;
4973         u32 vfta, index;
4974
4975         if ((hw->mng_cookie.status &
4976              E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
4977             (vid == adapter->mng_vlan_id))
4978                 return 0;
4979
4980         if (!e1000_vlan_used(adapter))
4981                 e1000_vlan_filter_on_off(adapter, true);
4982
4983         /* add VID to filter table */
4984         index = (vid >> 5) & 0x7F;
4985         vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4986         vfta |= (1 << (vid & 0x1F));
4987         e1000_write_vfta(hw, index, vfta);
4988
4989         set_bit(vid, adapter->active_vlans);
4990
4991         return 0;
4992 }
4993
4994 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
4995                                   __be16 proto, u16 vid)
4996 {
4997         struct e1000_adapter *adapter = netdev_priv(netdev);
4998         struct e1000_hw *hw = &adapter->hw;
4999         u32 vfta, index;
5000
5001         if (!test_bit(__E1000_DOWN, &adapter->flags))
5002                 e1000_irq_disable(adapter);
5003         if (!test_bit(__E1000_DOWN, &adapter->flags))
5004                 e1000_irq_enable(adapter);
5005
5006         /* remove VID from filter table */
5007         index = (vid >> 5) & 0x7F;
5008         vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
5009         vfta &= ~(1 << (vid & 0x1F));
5010         e1000_write_vfta(hw, index, vfta);
5011
5012         clear_bit(vid, adapter->active_vlans);
5013
5014         if (!e1000_vlan_used(adapter))
5015                 e1000_vlan_filter_on_off(adapter, false);
5016
5017         return 0;
5018 }
5019
5020 static void e1000_restore_vlan(struct e1000_adapter *adapter)
5021 {
5022         u16 vid;
5023
5024         if (!e1000_vlan_used(adapter))
5025                 return;
5026
5027         e1000_vlan_filter_on_off(adapter, true);
5028         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
5029                 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
5030 }
5031
5032 int e1000_set_spd_dplx(struct e1000_adapter *adapter, u32 spd, u8 dplx)
5033 {
5034         struct e1000_hw *hw = &adapter->hw;
5035
5036         hw->autoneg = 0;
5037
5038         /* Make sure dplx is at most 1 bit and lsb of speed is not set
5039          * for the switch() below to work
5040          */
5041         if ((spd & 1) || (dplx & ~1))
5042                 goto err_inval;
5043
5044         /* Fiber NICs only allow 1000 gbps Full duplex */
5045         if ((hw->media_type == e1000_media_type_fiber) &&
5046             spd != SPEED_1000 &&
5047             dplx != DUPLEX_FULL)
5048                 goto err_inval;
5049
5050         switch (spd + dplx) {
5051         case SPEED_10 + DUPLEX_HALF:
5052                 hw->forced_speed_duplex = e1000_10_half;
5053                 break;
5054         case SPEED_10 + DUPLEX_FULL:
5055                 hw->forced_speed_duplex = e1000_10_full;
5056                 break;
5057         case SPEED_100 + DUPLEX_HALF:
5058                 hw->forced_speed_duplex = e1000_100_half;
5059                 break;
5060         case SPEED_100 + DUPLEX_FULL:
5061                 hw->forced_speed_duplex = e1000_100_full;
5062                 break;
5063         case SPEED_1000 + DUPLEX_FULL:
5064                 hw->autoneg = 1;
5065                 hw->autoneg_advertised = ADVERTISE_1000_FULL;
5066                 break;
5067         case SPEED_1000 + DUPLEX_HALF: /* not supported */
5068         default:
5069                 goto err_inval;
5070         }
5071
5072         /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
5073         hw->mdix = AUTO_ALL_MODES;
5074
5075         return 0;
5076
5077 err_inval:
5078         e_err(probe, "Unsupported Speed/Duplex configuration\n");
5079         return -EINVAL;
5080 }
5081
5082 static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake)
5083 {
5084         struct net_device *netdev = pci_get_drvdata(pdev);
5085         struct e1000_adapter *adapter = netdev_priv(netdev);
5086         struct e1000_hw *hw = &adapter->hw;
5087         u32 ctrl, ctrl_ext, rctl, status;
5088         u32 wufc = adapter->wol;
5089 #ifdef CONFIG_PM
5090         int retval = 0;
5091 #endif
5092
5093         netif_device_detach(netdev);
5094
5095         if (netif_running(netdev)) {
5096                 int count = E1000_CHECK_RESET_COUNT;
5097
5098                 while (test_bit(__E1000_RESETTING, &adapter->flags) && count--)
5099                         usleep_range(10000, 20000);
5100
5101                 WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
5102                 e1000_down(adapter);
5103         }
5104
5105 #ifdef CONFIG_PM
5106         retval = pci_save_state(pdev);
5107         if (retval)
5108                 return retval;
5109 #endif
5110
5111         status = er32(STATUS);
5112         if (status & E1000_STATUS_LU)
5113                 wufc &= ~E1000_WUFC_LNKC;
5114
5115         if (wufc) {
5116                 e1000_setup_rctl(adapter);
5117                 e1000_set_rx_mode(netdev);
5118
5119                 rctl = er32(RCTL);
5120
5121                 /* turn on all-multi mode if wake on multicast is enabled */
5122                 if (wufc & E1000_WUFC_MC)
5123                         rctl |= E1000_RCTL_MPE;
5124
5125                 /* enable receives in the hardware */
5126                 ew32(RCTL, rctl | E1000_RCTL_EN);
5127
5128                 if (hw->mac_type >= e1000_82540) {
5129                         ctrl = er32(CTRL);
5130                         /* advertise wake from D3Cold */
5131                         #define E1000_CTRL_ADVD3WUC 0x00100000
5132                         /* phy power management enable */
5133                         #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5134                         ctrl |= E1000_CTRL_ADVD3WUC |
5135                                 E1000_CTRL_EN_PHY_PWR_MGMT;
5136                         ew32(CTRL, ctrl);
5137                 }
5138
5139                 if (hw->media_type == e1000_media_type_fiber ||
5140                     hw->media_type == e1000_media_type_internal_serdes) {
5141                         /* keep the laser running in D3 */
5142                         ctrl_ext = er32(CTRL_EXT);
5143                         ctrl_ext |= E1000_CTRL_EXT_SDP7_DATA;
5144                         ew32(CTRL_EXT, ctrl_ext);
5145                 }
5146
5147                 ew32(WUC, E1000_WUC_PME_EN);
5148                 ew32(WUFC, wufc);
5149         } else {
5150                 ew32(WUC, 0);
5151                 ew32(WUFC, 0);
5152         }
5153
5154         e1000_release_manageability(adapter);
5155
5156         *enable_wake = !!wufc;
5157
5158         /* make sure adapter isn't asleep if manageability is enabled */
5159         if (adapter->en_mng_pt)
5160                 *enable_wake = true;
5161
5162         if (netif_running(netdev))
5163                 e1000_free_irq(adapter);
5164
5165         if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags))
5166                 pci_disable_device(pdev);
5167
5168         return 0;
5169 }
5170
5171 #ifdef CONFIG_PM
5172 static int e1000_suspend(struct pci_dev *pdev, pm_message_t state)
5173 {
5174         int retval;
5175         bool wake;
5176
5177         retval = __e1000_shutdown(pdev, &wake);
5178         if (retval)
5179                 return retval;
5180
5181         if (wake) {
5182                 pci_prepare_to_sleep(pdev);
5183         } else {
5184                 pci_wake_from_d3(pdev, false);
5185                 pci_set_power_state(pdev, PCI_D3hot);
5186         }
5187
5188         return 0;
5189 }
5190
5191 static int e1000_resume(struct pci_dev *pdev)
5192 {
5193         struct net_device *netdev = pci_get_drvdata(pdev);
5194         struct e1000_adapter *adapter = netdev_priv(netdev);
5195         struct e1000_hw *hw = &adapter->hw;
5196         u32 err;
5197
5198         pci_set_power_state(pdev, PCI_D0);
5199         pci_restore_state(pdev);
5200         pci_save_state(pdev);
5201
5202         if (adapter->need_ioport)
5203                 err = pci_enable_device(pdev);
5204         else
5205                 err = pci_enable_device_mem(pdev);
5206         if (err) {
5207                 pr_err("Cannot enable PCI device from suspend\n");
5208                 return err;
5209         }
5210
5211         /* flush memory to make sure state is correct */
5212         smp_mb__before_atomic();
5213         clear_bit(__E1000_DISABLED, &adapter->flags);
5214         pci_set_master(pdev);
5215
5216         pci_enable_wake(pdev, PCI_D3hot, 0);
5217         pci_enable_wake(pdev, PCI_D3cold, 0);
5218
5219         if (netif_running(netdev)) {
5220                 err = e1000_request_irq(adapter);
5221                 if (err)
5222                         return err;
5223         }
5224
5225         e1000_power_up_phy(adapter);
5226         e1000_reset(adapter);
5227         ew32(WUS, ~0);
5228
5229         e1000_init_manageability(adapter);
5230
5231         if (netif_running(netdev))
5232                 e1000_up(adapter);
5233
5234         netif_device_attach(netdev);
5235
5236         return 0;
5237 }
5238 #endif
5239
5240 static void e1000_shutdown(struct pci_dev *pdev)
5241 {
5242         bool wake;
5243
5244         __e1000_shutdown(pdev, &wake);
5245
5246         if (system_state == SYSTEM_POWER_OFF) {
5247                 pci_wake_from_d3(pdev, wake);
5248                 pci_set_power_state(pdev, PCI_D3hot);
5249         }
5250 }
5251
5252 #ifdef CONFIG_NET_POLL_CONTROLLER
5253 /* Polling 'interrupt' - used by things like netconsole to send skbs
5254  * without having to re-enable interrupts. It's not called while
5255  * the interrupt routine is executing.
5256  */
5257 static void e1000_netpoll(struct net_device *netdev)
5258 {
5259         struct e1000_adapter *adapter = netdev_priv(netdev);
5260
5261         if (disable_hardirq(adapter->pdev->irq))
5262                 e1000_intr(adapter->pdev->irq, netdev);
5263         enable_irq(adapter->pdev->irq);
5264 }
5265 #endif
5266
5267 /**
5268  * e1000_io_error_detected - called when PCI error is detected
5269  * @pdev: Pointer to PCI device
5270  * @state: The current pci connection state
5271  *
5272  * This function is called after a PCI bus error affecting
5273  * this device has been detected.
5274  */
5275 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
5276                                                 pci_channel_state_t state)
5277 {
5278         struct net_device *netdev = pci_get_drvdata(pdev);
5279         struct e1000_adapter *adapter = netdev_priv(netdev);
5280
5281         netif_device_detach(netdev);
5282
5283         if (state == pci_channel_io_perm_failure)
5284                 return PCI_ERS_RESULT_DISCONNECT;
5285
5286         if (netif_running(netdev))
5287                 e1000_down(adapter);
5288
5289         if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags))
5290                 pci_disable_device(pdev);
5291
5292         /* Request a slot slot reset. */
5293         return PCI_ERS_RESULT_NEED_RESET;
5294 }
5295
5296 /**
5297  * e1000_io_slot_reset - called after the pci bus has been reset.
5298  * @pdev: Pointer to PCI device
5299  *
5300  * Restart the card from scratch, as if from a cold-boot. Implementation
5301  * resembles the first-half of the e1000_resume routine.
5302  */
5303 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
5304 {
5305         struct net_device *netdev = pci_get_drvdata(pdev);
5306         struct e1000_adapter *adapter = netdev_priv(netdev);
5307         struct e1000_hw *hw = &adapter->hw;
5308         int err;
5309
5310         if (adapter->need_ioport)
5311                 err = pci_enable_device(pdev);
5312         else
5313                 err = pci_enable_device_mem(pdev);
5314         if (err) {
5315                 pr_err("Cannot re-enable PCI device after reset.\n");
5316                 return PCI_ERS_RESULT_DISCONNECT;
5317         }
5318
5319         /* flush memory to make sure state is correct */
5320         smp_mb__before_atomic();
5321         clear_bit(__E1000_DISABLED, &adapter->flags);
5322         pci_set_master(pdev);
5323
5324         pci_enable_wake(pdev, PCI_D3hot, 0);
5325         pci_enable_wake(pdev, PCI_D3cold, 0);
5326
5327         e1000_reset(adapter);
5328         ew32(WUS, ~0);
5329
5330         return PCI_ERS_RESULT_RECOVERED;
5331 }
5332
5333 /**
5334  * e1000_io_resume - called when traffic can start flowing again.
5335  * @pdev: Pointer to PCI device
5336  *
5337  * This callback is called when the error recovery driver tells us that
5338  * its OK to resume normal operation. Implementation resembles the
5339  * second-half of the e1000_resume routine.
5340  */
5341 static void e1000_io_resume(struct pci_dev *pdev)
5342 {
5343         struct net_device *netdev = pci_get_drvdata(pdev);
5344         struct e1000_adapter *adapter = netdev_priv(netdev);
5345
5346         e1000_init_manageability(adapter);
5347
5348         if (netif_running(netdev)) {
5349                 if (e1000_up(adapter)) {
5350                         pr_info("can't bring device back up after reset\n");
5351                         return;
5352                 }
5353         }
5354
5355         netif_device_attach(netdev);
5356 }
5357
5358 /* e1000_main.c */