drivers/net/ethernet/intel/e1000/e1000_main.c

   1 /*******************************************************************************
   2
   3   Intel PRO/1000 Linux driver
   4   Copyright(c) 1999 - 2006 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   Linux NICS <linux.nics@intel.com>
  24   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  25   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  26
  27 *******************************************************************************/
  28
  29 #include "e1000.h"
  30 #include <net/ip6_checksum.h>
  31 #include <linux/io.h>
  32 #include <linux/prefetch.h>
  33 #include <linux/bitops.h>
  34 #include <linux/if_vlan.h>
  35
  36 char e1000_driver_name[] = "e1000";
  37 static char e1000_driver_string[] = "Intel(R) PRO/1000 Network Driver";
  38 #define DRV_VERSION "7.3.21-k8-NAPI"
  39 const char e1000_driver_version[] = DRV_VERSION;
  40 static const char e1000_copyright[] = "Copyright (c) 1999-2006 Intel Corporation.";
  41
  42 /* e1000_pci_tbl - PCI Device ID Table
  43  *
  44  * Last entry must be all 0s
  45  *
  46  * Macro expands to...
  47  *   {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
  48  */
  49 static const struct pci_device_id e1000_pci_tbl[] = {
  50         INTEL_E1000_ETHERNET_DEVICE(0x1000),
  51         INTEL_E1000_ETHERNET_DEVICE(0x1001),
  52         INTEL_E1000_ETHERNET_DEVICE(0x1004),
  53         INTEL_E1000_ETHERNET_DEVICE(0x1008),
  54         INTEL_E1000_ETHERNET_DEVICE(0x1009),
  55         INTEL_E1000_ETHERNET_DEVICE(0x100C),
  56         INTEL_E1000_ETHERNET_DEVICE(0x100D),
  57         INTEL_E1000_ETHERNET_DEVICE(0x100E),
  58         INTEL_E1000_ETHERNET_DEVICE(0x100F),
  59         INTEL_E1000_ETHERNET_DEVICE(0x1010),
  60         INTEL_E1000_ETHERNET_DEVICE(0x1011),
  61         INTEL_E1000_ETHERNET_DEVICE(0x1012),
  62         INTEL_E1000_ETHERNET_DEVICE(0x1013),
  63         INTEL_E1000_ETHERNET_DEVICE(0x1014),
  64         INTEL_E1000_ETHERNET_DEVICE(0x1015),
  65         INTEL_E1000_ETHERNET_DEVICE(0x1016),
  66         INTEL_E1000_ETHERNET_DEVICE(0x1017),
  67         INTEL_E1000_ETHERNET_DEVICE(0x1018),
  68         INTEL_E1000_ETHERNET_DEVICE(0x1019),
  69         INTEL_E1000_ETHERNET_DEVICE(0x101A),
  70         INTEL_E1000_ETHERNET_DEVICE(0x101D),
  71         INTEL_E1000_ETHERNET_DEVICE(0x101E),
  72         INTEL_E1000_ETHERNET_DEVICE(0x1026),
  73         INTEL_E1000_ETHERNET_DEVICE(0x1027),
  74         INTEL_E1000_ETHERNET_DEVICE(0x1028),
  75         INTEL_E1000_ETHERNET_DEVICE(0x1075),
  76         INTEL_E1000_ETHERNET_DEVICE(0x1076),
  77         INTEL_E1000_ETHERNET_DEVICE(0x1077),
  78         INTEL_E1000_ETHERNET_DEVICE(0x1078),
  79         INTEL_E1000_ETHERNET_DEVICE(0x1079),
  80         INTEL_E1000_ETHERNET_DEVICE(0x107A),
  81         INTEL_E1000_ETHERNET_DEVICE(0x107B),
  82         INTEL_E1000_ETHERNET_DEVICE(0x107C),
  83         INTEL_E1000_ETHERNET_DEVICE(0x108A),
  84         INTEL_E1000_ETHERNET_DEVICE(0x1099),
  85         INTEL_E1000_ETHERNET_DEVICE(0x10B5),
  86         INTEL_E1000_ETHERNET_DEVICE(0x2E6E),
  87         /* required last entry */
  88         {0,}
  89 };
  90
  91 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
  92
  93 int e1000_up(struct e1000_adapter *adapter);
  94 void e1000_down(struct e1000_adapter *adapter);
  95 void e1000_reinit_locked(struct e1000_adapter *adapter);
  96 void e1000_reset(struct e1000_adapter *adapter);
  97 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter);
  98 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter);
  99 void e1000_free_all_tx_resources(struct e1000_adapter *adapter);
 100 void e1000_free_all_rx_resources(struct e1000_adapter *adapter);
 101 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
 102                                     struct e1000_tx_ring *txdr);
 103 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
 104                                     struct e1000_rx_ring *rxdr);
 105 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
 106                                     struct e1000_tx_ring *tx_ring);
 107 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
 108                                     struct e1000_rx_ring *rx_ring);
 109 void e1000_update_stats(struct e1000_adapter *adapter);
 110
 111 static int e1000_init_module(void);
 112 static void e1000_exit_module(void);
 113 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
 114 static void e1000_remove(struct pci_dev *pdev);
 115 static int e1000_alloc_queues(struct e1000_adapter *adapter);
 116 static int e1000_sw_init(struct e1000_adapter *adapter);
 117 int e1000_open(struct net_device *netdev);
 118 int e1000_close(struct net_device *netdev);
 119 static void e1000_configure_tx(struct e1000_adapter *adapter);
 120 static void e1000_configure_rx(struct e1000_adapter *adapter);
 121 static void e1000_setup_rctl(struct e1000_adapter *adapter);
 122 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter);
 123 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter);
 124 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
 125                                 struct e1000_tx_ring *tx_ring);
 126 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
 127                                 struct e1000_rx_ring *rx_ring);
 128 static void e1000_set_rx_mode(struct net_device *netdev);
 129 static void e1000_update_phy_info_task(struct work_struct *work);
 130 static void e1000_watchdog(struct work_struct *work);
 131 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work);
 132 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
 133                                     struct net_device *netdev);
 134 static int e1000_change_mtu(struct net_device *netdev, int new_mtu);
 135 static int e1000_set_mac(struct net_device *netdev, void *p);
 136 static irqreturn_t e1000_intr(int irq, void *data);
 137 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
 138                                struct e1000_tx_ring *tx_ring);
 139 static int e1000_clean(struct napi_struct *napi, int budget);
 140 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
 141                                struct e1000_rx_ring *rx_ring,
 142                                int *work_done, int work_to_do);
 143 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
 144                                      struct e1000_rx_ring *rx_ring,
 145                                      int *work_done, int work_to_do);
 146 static void e1000_alloc_dummy_rx_buffers(struct e1000_adapter *adapter,
 147                                          struct e1000_rx_ring *rx_ring,
 148                                          int cleaned_count)
 149 {
 150 }
 151 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
 152                                    struct e1000_rx_ring *rx_ring,
 153                                    int cleaned_count);
 154 static void e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
 155                                          struct e1000_rx_ring *rx_ring,
 156                                          int cleaned_count);
 157 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd);
 158 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
 159                            int cmd);
 160 static void e1000_enter_82542_rst(struct e1000_adapter *adapter);
 161 static void e1000_leave_82542_rst(struct e1000_adapter *adapter);
 162 static void e1000_tx_timeout(struct net_device *dev);
 163 static void e1000_reset_task(struct work_struct *work);
 164 static void e1000_smartspeed(struct e1000_adapter *adapter);
 165 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
 166                                        struct sk_buff *skb);
 167
 168 static bool e1000_vlan_used(struct e1000_adapter *adapter);
 169 static void e1000_vlan_mode(struct net_device *netdev,
 170                             netdev_features_t features);
 171 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
 172                                      bool filter_on);
 173 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
 174                                  __be16 proto, u16 vid);
 175 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
 176                                   __be16 proto, u16 vid);
 177 static void e1000_restore_vlan(struct e1000_adapter *adapter);
 178
 179 #ifdef CONFIG_PM
 180 static int e1000_suspend(struct pci_dev *pdev, pm_message_t state);
 181 static int e1000_resume(struct pci_dev *pdev);
 182 #endif
 183 static void e1000_shutdown(struct pci_dev *pdev);
 184
 185 #ifdef CONFIG_NET_POLL_CONTROLLER
 186 /* for netdump / net console */
 187 static void e1000_netpoll (struct net_device *netdev);
 188 #endif
 189
 190 #define COPYBREAK_DEFAULT 256
 191 static unsigned int copybreak __read_mostly = COPYBREAK_DEFAULT;
 192 module_param(copybreak, uint, 0644);
 193 MODULE_PARM_DESC(copybreak,
 194         "Maximum size of packet that is copied to a new buffer on receive");
 195
 196 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
 197                                                 pci_channel_state_t state);
 198 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev);
 199 static void e1000_io_resume(struct pci_dev *pdev);
 200
 201 static const struct pci_error_handlers e1000_err_handler = {
 202         .error_detected = e1000_io_error_detected,
 203         .slot_reset = e1000_io_slot_reset,
 204         .resume = e1000_io_resume,
 205 };
 206
 207 static struct pci_driver e1000_driver = {
 208         .name     = e1000_driver_name,
 209         .id_table = e1000_pci_tbl,
 210         .probe    = e1000_probe,
 211         .remove   = e1000_remove,
 212 #ifdef CONFIG_PM
 213         /* Power Management Hooks */
 214         .suspend  = e1000_suspend,
 215         .resume   = e1000_resume,
 216 #endif
 217         .shutdown = e1000_shutdown,
 218         .err_handler = &e1000_err_handler
 219 };
 220
 221 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
 222 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
 223 MODULE_LICENSE("GPL");
 224 MODULE_VERSION(DRV_VERSION);
 225
 226 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
 227 static int debug = -1;
 228 module_param(debug, int, 0);
 229 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 230
 231 /**
 232  * e1000_get_hw_dev - return device
 233  * used by hardware layer to print debugging information
 234  *
 235  **/
 236 struct net_device *e1000_get_hw_dev(struct e1000_hw *hw)
 237 {
 238         struct e1000_adapter *adapter = hw->back;
 239         return adapter->netdev;
 240 }
 241
 242 /**
 243  * e1000_init_module - Driver Registration Routine
 244  *
 245  * e1000_init_module is the first routine called when the driver is
 246  * loaded. All it does is register with the PCI subsystem.
 247  **/
 248 static int __init e1000_init_module(void)
 249 {
 250         int ret;
 251         pr_info("%s - version %s\n", e1000_driver_string, e1000_driver_version);
 252
 253         pr_info("%s\n", e1000_copyright);
 254
 255         ret = pci_register_driver(&e1000_driver);
 256         if (copybreak != COPYBREAK_DEFAULT) {
 257                 if (copybreak == 0)
 258                         pr_info("copybreak disabled\n");
 259                 else
 260                         pr_info("copybreak enabled for "
 261                                    "packets <= %u bytes\n", copybreak);
 262         }
 263         return ret;
 264 }
 265
 266 module_init(e1000_init_module);
 267
 268 /**
 269  * e1000_exit_module - Driver Exit Cleanup Routine
 270  *
 271  * e1000_exit_module is called just before the driver is removed
 272  * from memory.
 273  **/
 274 static void __exit e1000_exit_module(void)
 275 {
 276         pci_unregister_driver(&e1000_driver);
 277 }
 278
 279 module_exit(e1000_exit_module);
 280
 281 static int e1000_request_irq(struct e1000_adapter *adapter)
 282 {
 283         struct net_device *netdev = adapter->netdev;
 284         irq_handler_t handler = e1000_intr;
 285         int irq_flags = IRQF_SHARED;
 286         int err;
 287
 288         err = request_irq(adapter->pdev->irq, handler, irq_flags, netdev->name,
 289                           netdev);
 290         if (err) {
 291                 e_err(probe, "Unable to allocate interrupt Error: %d\n", err);
 292         }
 293
 294         return err;
 295 }
 296
 297 static void e1000_free_irq(struct e1000_adapter *adapter)
 298 {
 299         struct net_device *netdev = adapter->netdev;
 300
 301         free_irq(adapter->pdev->irq, netdev);
 302 }
 303
 304 /**
 305  * e1000_irq_disable - Mask off interrupt generation on the NIC
 306  * @adapter: board private structure
 307  **/
 308 static void e1000_irq_disable(struct e1000_adapter *adapter)
 309 {
 310         struct e1000_hw *hw = &adapter->hw;
 311
 312         ew32(IMC, ~0);
 313         E1000_WRITE_FLUSH();
 314         synchronize_irq(adapter->pdev->irq);
 315 }
 316
 317 /**
 318  * e1000_irq_enable - Enable default interrupt generation settings
 319  * @adapter: board private structure
 320  **/
 321 static void e1000_irq_enable(struct e1000_adapter *adapter)
 322 {
 323         struct e1000_hw *hw = &adapter->hw;
 324
 325         ew32(IMS, IMS_ENABLE_MASK);
 326         E1000_WRITE_FLUSH();
 327 }
 328
 329 static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
 330 {
 331         struct e1000_hw *hw = &adapter->hw;
 332         struct net_device *netdev = adapter->netdev;
 333         u16 vid = hw->mng_cookie.vlan_id;
 334         u16 old_vid = adapter->mng_vlan_id;
 335
 336         if (!e1000_vlan_used(adapter))
 337                 return;
 338
 339         if (!test_bit(vid, adapter->active_vlans)) {
 340                 if (hw->mng_cookie.status &
 341                     E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) {
 342                         e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
 343                         adapter->mng_vlan_id = vid;
 344                 } else {
 345                         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
 346                 }
 347                 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) &&
 348                     (vid != old_vid) &&
 349                     !test_bit(old_vid, adapter->active_vlans))
 350                         e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
 351                                                old_vid);
 352         } else {
 353                 adapter->mng_vlan_id = vid;
 354         }
 355 }
 356
 357 static void e1000_init_manageability(struct e1000_adapter *adapter)
 358 {
 359         struct e1000_hw *hw = &adapter->hw;
 360
 361         if (adapter->en_mng_pt) {
 362                 u32 manc = er32(MANC);
 363
 364                 /* disable hardware interception of ARP */
 365                 manc &= ~(E1000_MANC_ARP_EN);
 366
 367                 ew32(MANC, manc);
 368         }
 369 }
 370
 371 static void e1000_release_manageability(struct e1000_adapter *adapter)
 372 {
 373         struct e1000_hw *hw = &adapter->hw;
 374
 375         if (adapter->en_mng_pt) {
 376                 u32 manc = er32(MANC);
 377
 378                 /* re-enable hardware interception of ARP */
 379                 manc |= E1000_MANC_ARP_EN;
 380
 381                 ew32(MANC, manc);
 382         }
 383 }
 384
 385 /**
 386  * e1000_configure - configure the hardware for RX and TX
 387  * @adapter = private board structure
 388  **/
 389 static void e1000_configure(struct e1000_adapter *adapter)
 390 {
 391         struct net_device *netdev = adapter->netdev;
 392         int i;
 393
 394         e1000_set_rx_mode(netdev);
 395
 396         e1000_restore_vlan(adapter);
 397         e1000_init_manageability(adapter);
 398
 399         e1000_configure_tx(adapter);
 400         e1000_setup_rctl(adapter);
 401         e1000_configure_rx(adapter);
 402         /* call E1000_DESC_UNUSED which always leaves
 403          * at least 1 descriptor unused to make sure
 404          * next_to_use != next_to_clean
 405          */
 406         for (i = 0; i < adapter->num_rx_queues; i++) {
 407                 struct e1000_rx_ring *ring = &adapter->rx_ring[i];
 408                 adapter->alloc_rx_buf(adapter, ring,
 409                                       E1000_DESC_UNUSED(ring));
 410         }
 411 }
 412
 413 int e1000_up(struct e1000_adapter *adapter)
 414 {
 415         struct e1000_hw *hw = &adapter->hw;
 416
 417         /* hardware has been reset, we need to reload some things */
 418         e1000_configure(adapter);
 419
 420         clear_bit(__E1000_DOWN, &adapter->flags);
 421
 422         napi_enable(&adapter->napi);
 423
 424         e1000_irq_enable(adapter);
 425
 426         netif_wake_queue(adapter->netdev);
 427
 428         /* fire a link change interrupt to start the watchdog */
 429         ew32(ICS, E1000_ICS_LSC);
 430         return 0;
 431 }
 432
 433 /**
 434  * e1000_power_up_phy - restore link in case the phy was powered down
 435  * @adapter: address of board private structure
 436  *
 437  * The phy may be powered down to save power and turn off link when the
 438  * driver is unloaded and wake on lan is not enabled (among others)
 439  * *** this routine MUST be followed by a call to e1000_reset ***
 440  **/
 441 void e1000_power_up_phy(struct e1000_adapter *adapter)
 442 {
 443         struct e1000_hw *hw = &adapter->hw;
 444         u16 mii_reg = 0;
 445
 446         /* Just clear the power down bit to wake the phy back up */
 447         if (hw->media_type == e1000_media_type_copper) {
 448                 /* according to the manual, the phy will retain its
 449                  * settings across a power-down/up cycle
 450                  */
 451                 e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
 452                 mii_reg &= ~MII_CR_POWER_DOWN;
 453                 e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
 454         }
 455 }
 456
 457 static void e1000_power_down_phy(struct e1000_adapter *adapter)
 458 {
 459         struct e1000_hw *hw = &adapter->hw;
 460
 461         /* Power down the PHY so no link is implied when interface is down *
 462          * The PHY cannot be powered down if any of the following is true *
 463          * (a) WoL is enabled
 464          * (b) AMT is active
 465          * (c) SoL/IDER session is active
 466          */
 467         if (!adapter->wol && hw->mac_type >= e1000_82540 &&
 468            hw->media_type == e1000_media_type_copper) {
 469                 u16 mii_reg = 0;
 470
 471                 switch (hw->mac_type) {
 472                 case e1000_82540:
 473                 case e1000_82545:
 474                 case e1000_82545_rev_3:
 475                 case e1000_82546:
 476                 case e1000_ce4100:
 477                 case e1000_82546_rev_3:
 478                 case e1000_82541:
 479                 case e1000_82541_rev_2:
 480                 case e1000_82547:
 481                 case e1000_82547_rev_2:
 482                         if (er32(MANC) & E1000_MANC_SMBUS_EN)
 483                                 goto out;
 484                         break;
 485                 default:
 486                         goto out;
 487                 }
 488                 e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
 489                 mii_reg |= MII_CR_POWER_DOWN;
 490                 e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
 491                 msleep(1);
 492         }
 493 out:
 494         return;
 495 }
 496
 497 static void e1000_down_and_stop(struct e1000_adapter *adapter)
 498 {
 499         set_bit(__E1000_DOWN, &adapter->flags);
 500
 501         cancel_delayed_work_sync(&adapter->watchdog_task);
 502
 503         /*
 504          * Since the watchdog task can reschedule other tasks, we should cancel
 505          * it first, otherwise we can run into the situation when a work is
 506          * still running after the adapter has been turned down.
 507          */
 508
 509         cancel_delayed_work_sync(&adapter->phy_info_task);
 510         cancel_delayed_work_sync(&adapter->fifo_stall_task);
 511
 512         /* Only kill reset task if adapter is not resetting */
 513         if (!test_bit(__E1000_RESETTING, &adapter->flags))
 514                 cancel_work_sync(&adapter->reset_task);
 515 }
 516
 517 void e1000_down(struct e1000_adapter *adapter)
 518 {
 519         struct e1000_hw *hw = &adapter->hw;
 520         struct net_device *netdev = adapter->netdev;
 521         u32 rctl, tctl;
 522
 523         /* disable receives in the hardware */
 524         rctl = er32(RCTL);
 525         ew32(RCTL, rctl & ~E1000_RCTL_EN);
 526         /* flush and sleep below */
 527
 528         netif_tx_disable(netdev);
 529
 530         /* disable transmits in the hardware */
 531         tctl = er32(TCTL);
 532         tctl &= ~E1000_TCTL_EN;
 533         ew32(TCTL, tctl);
 534         /* flush both disables and wait for them to finish */
 535         E1000_WRITE_FLUSH();
 536         msleep(10);
 537
 538         /* Set the carrier off after transmits have been disabled in the
 539          * hardware, to avoid race conditions with e1000_watchdog() (which
 540          * may be running concurrently to us, checking for the carrier
 541          * bit to decide whether it should enable transmits again). Such
 542          * a race condition would result into transmission being disabled
 543          * in the hardware until the next IFF_DOWN+IFF_UP cycle.
 544          */
 545         netif_carrier_off(netdev);
 546
 547         napi_disable(&adapter->napi);
 548
 549         e1000_irq_disable(adapter);
 550
 551         /* Setting DOWN must be after irq_disable to prevent
 552          * a screaming interrupt.  Setting DOWN also prevents
 553          * tasks from rescheduling.
 554          */
 555         e1000_down_and_stop(adapter);
 556
 557         adapter->link_speed = 0;
 558         adapter->link_duplex = 0;
 559
 560         e1000_reset(adapter);
 561         e1000_clean_all_tx_rings(adapter);
 562         e1000_clean_all_rx_rings(adapter);
 563 }
 564
 565 void e1000_reinit_locked(struct e1000_adapter *adapter)
 566 {
 567         WARN_ON(in_interrupt());
 568         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
 569                 msleep(1);
 570         e1000_down(adapter);
 571         e1000_up(adapter);
 572         clear_bit(__E1000_RESETTING, &adapter->flags);
 573 }
 574
 575 void e1000_reset(struct e1000_adapter *adapter)
 576 {
 577         struct e1000_hw *hw = &adapter->hw;
 578         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
 579         bool legacy_pba_adjust = false;
 580         u16 hwm;
 581
 582         /* Repartition Pba for greater than 9k mtu
 583          * To take effect CTRL.RST is required.
 584          */
 585
 586         switch (hw->mac_type) {
 587         case e1000_82542_rev2_0:
 588         case e1000_82542_rev2_1:
 589         case e1000_82543:
 590         case e1000_82544:
 591         case e1000_82540:
 592         case e1000_82541:
 593         case e1000_82541_rev_2:
 594                 legacy_pba_adjust = true;
 595                 pba = E1000_PBA_48K;
 596                 break;
 597         case e1000_82545:
 598         case e1000_82545_rev_3:
 599         case e1000_82546:
 600         case e1000_ce4100:
 601         case e1000_82546_rev_3:
 602                 pba = E1000_PBA_48K;
 603                 break;
 604         case e1000_82547:
 605         case e1000_82547_rev_2:
 606                 legacy_pba_adjust = true;
 607                 pba = E1000_PBA_30K;
 608                 break;
 609         case e1000_undefined:
 610         case e1000_num_macs:
 611                 break;
 612         }
 613
 614         if (legacy_pba_adjust) {
 615                 if (hw->max_frame_size > E1000_RXBUFFER_8192)
 616                         pba -= 8; /* allocate more FIFO for Tx */
 617
 618                 if (hw->mac_type == e1000_82547) {
 619                         adapter->tx_fifo_head = 0;
 620                         adapter->tx_head_addr = pba << E1000_TX_HEAD_ADDR_SHIFT;
 621                         adapter->tx_fifo_size =
 622                                 (E1000_PBA_40K - pba) << E1000_PBA_BYTES_SHIFT;
 623                         atomic_set(&adapter->tx_fifo_stall, 0);
 624                 }
 625         } else if (hw->max_frame_size >  ETH_FRAME_LEN + ETH_FCS_LEN) {
 626                 /* adjust PBA for jumbo frames */
 627                 ew32(PBA, pba);
 628
 629                 /* To maintain wire speed transmits, the Tx FIFO should be
 630                  * large enough to accommodate two full transmit packets,
 631                  * rounded up to the next 1KB and expressed in KB.  Likewise,
 632                  * the Rx FIFO should be large enough to accommodate at least
 633                  * one full receive packet and is similarly rounded up and
 634                  * expressed in KB.
 635                  */
 636                 pba = er32(PBA);
 637                 /* upper 16 bits has Tx packet buffer allocation size in KB */
 638                 tx_space = pba >> 16;
 639                 /* lower 16 bits has Rx packet buffer allocation size in KB */
 640                 pba &= 0xffff;
 641                 /* the Tx fifo also stores 16 bytes of information about the Tx
 642                  * but don't include ethernet FCS because hardware appends it
 643                  */
 644                 min_tx_space = (hw->max_frame_size +
 645                                 sizeof(struct e1000_tx_desc) -
 646                                 ETH_FCS_LEN) * 2;
 647                 min_tx_space = ALIGN(min_tx_space, 1024);
 648                 min_tx_space >>= 10;
 649                 /* software strips receive CRC, so leave room for it */
 650                 min_rx_space = hw->max_frame_size;
 651                 min_rx_space = ALIGN(min_rx_space, 1024);
 652                 min_rx_space >>= 10;
 653
 654                 /* If current Tx allocation is less than the min Tx FIFO size,
 655                  * and the min Tx FIFO size is less than the current Rx FIFO
 656                  * allocation, take space away from current Rx allocation
 657                  */
 658                 if (tx_space < min_tx_space &&
 659                     ((min_tx_space - tx_space) < pba)) {
 660                         pba = pba - (min_tx_space - tx_space);
 661
 662                         /* PCI/PCIx hardware has PBA alignment constraints */
 663                         switch (hw->mac_type) {
 664                         case e1000_82545 ... e1000_82546_rev_3:
 665                                 pba &= ~(E1000_PBA_8K - 1);
 666                                 break;
 667                         default:
 668                                 break;
 669                         }
 670
 671                         /* if short on Rx space, Rx wins and must trump Tx
 672                          * adjustment or use Early Receive if available
 673                          */
 674                         if (pba < min_rx_space)
 675                                 pba = min_rx_space;
 676                 }
 677         }
 678
 679         ew32(PBA, pba);
 680
 681         /* flow control settings:
 682          * The high water mark must be low enough to fit one full frame
 683          * (or the size used for early receive) above it in the Rx FIFO.
 684          * Set it to the lower of:
 685          * - 90% of the Rx FIFO size, and
 686          * - the full Rx FIFO size minus the early receive size (for parts
 687          *   with ERT support assuming ERT set to E1000_ERT_2048), or
 688          * - the full Rx FIFO size minus one full frame
 689          */
 690         hwm = min(((pba << 10) * 9 / 10),
 691                   ((pba << 10) - hw->max_frame_size));
 692
 693         hw->fc_high_water = hwm & 0xFFF8;       /* 8-byte granularity */
 694         hw->fc_low_water = hw->fc_high_water - 8;
 695         hw->fc_pause_time = E1000_FC_PAUSE_TIME;
 696         hw->fc_send_xon = 1;
 697         hw->fc = hw->original_fc;
 698
 699         /* Allow time for pending master requests to run */
 700         e1000_reset_hw(hw);
 701         if (hw->mac_type >= e1000_82544)
 702                 ew32(WUC, 0);
 703
 704         if (e1000_init_hw(hw))
 705                 e_dev_err("Hardware Error\n");
 706         e1000_update_mng_vlan(adapter);
 707
 708         /* if (adapter->hwflags & HWFLAGS_PHY_PWR_BIT) { */
 709         if (hw->mac_type >= e1000_82544 &&
 710             hw->autoneg == 1 &&
 711             hw->autoneg_advertised == ADVERTISE_1000_FULL) {
 712                 u32 ctrl = er32(CTRL);
 713                 /* clear phy power management bit if we are in gig only mode,
 714                  * which if enabled will attempt negotiation to 100Mb, which
 715                  * can cause a loss of link at power off or driver unload
 716                  */
 717                 ctrl &= ~E1000_CTRL_SWDPIN3;
 718                 ew32(CTRL, ctrl);
 719         }
 720
 721         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
 722         ew32(VET, ETHERNET_IEEE_VLAN_TYPE);
 723
 724         e1000_reset_adaptive(hw);
 725         e1000_phy_get_info(hw, &adapter->phy_info);
 726
 727         e1000_release_manageability(adapter);
 728 }
 729
 730 /* Dump the eeprom for users having checksum issues */
 731 static void e1000_dump_eeprom(struct e1000_adapter *adapter)
 732 {
 733         struct net_device *netdev = adapter->netdev;
 734         struct ethtool_eeprom eeprom;
 735         const struct ethtool_ops *ops = netdev->ethtool_ops;
 736         u8 *data;
 737         int i;
 738         u16 csum_old, csum_new = 0;
 739
 740         eeprom.len = ops->get_eeprom_len(netdev);
 741         eeprom.offset = 0;
 742
 743         data = kmalloc(eeprom.len, GFP_KERNEL);
 744         if (!data)
 745                 return;
 746
 747         ops->get_eeprom(netdev, &eeprom, data);
 748
 749         csum_old = (data[EEPROM_CHECKSUM_REG * 2]) +
 750                    (data[EEPROM_CHECKSUM_REG * 2 + 1] << 8);
 751         for (i = 0; i < EEPROM_CHECKSUM_REG * 2; i += 2)
 752                 csum_new += data[i] + (data[i + 1] << 8);
 753         csum_new = EEPROM_SUM - csum_new;
 754
 755         pr_err("/*********************/\n");
 756         pr_err("Current EEPROM Checksum : 0x%04x\n", csum_old);
 757         pr_err("Calculated              : 0x%04x\n", csum_new);
 758
 759         pr_err("Offset    Values\n");
 760         pr_err("========  ======\n");
 761         print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, data, 128, 0);
 762
 763         pr_err("Include this output when contacting your support provider.\n");
 764         pr_err("This is not a software error! Something bad happened to\n");
 765         pr_err("your hardware or EEPROM image. Ignoring this problem could\n");
 766         pr_err("result in further problems, possibly loss of data,\n");
 767         pr_err("corruption or system hangs!\n");
 768         pr_err("The MAC Address will be reset to 00:00:00:00:00:00,\n");
 769         pr_err("which is invalid and requires you to set the proper MAC\n");
 770         pr_err("address manually before continuing to enable this network\n");
 771         pr_err("device. Please inspect the EEPROM dump and report the\n");
 772         pr_err("issue to your hardware vendor or Intel Customer Support.\n");
 773         pr_err("/*********************/\n");
 774
 775         kfree(data);
 776 }
 777
 778 /**
 779  * e1000_is_need_ioport - determine if an adapter needs ioport resources or not
 780  * @pdev: PCI device information struct
 781  *
 782  * Return true if an adapter needs ioport resources
 783  **/
 784 static int e1000_is_need_ioport(struct pci_dev *pdev)
 785 {
 786         switch (pdev->device) {
 787         case E1000_DEV_ID_82540EM:
 788         case E1000_DEV_ID_82540EM_LOM:
 789         case E1000_DEV_ID_82540EP:
 790         case E1000_DEV_ID_82540EP_LOM:
 791         case E1000_DEV_ID_82540EP_LP:
 792         case E1000_DEV_ID_82541EI:
 793         case E1000_DEV_ID_82541EI_MOBILE:
 794         case E1000_DEV_ID_82541ER:
 795         case E1000_DEV_ID_82541ER_LOM:
 796         case E1000_DEV_ID_82541GI:
 797         case E1000_DEV_ID_82541GI_LF:
 798         case E1000_DEV_ID_82541GI_MOBILE:
 799         case E1000_DEV_ID_82544EI_COPPER:
 800         case E1000_DEV_ID_82544EI_FIBER:
 801         case E1000_DEV_ID_82544GC_COPPER:
 802         case E1000_DEV_ID_82544GC_LOM:
 803         case E1000_DEV_ID_82545EM_COPPER:
 804         case E1000_DEV_ID_82545EM_FIBER:
 805         case E1000_DEV_ID_82546EB_COPPER:
 806         case E1000_DEV_ID_82546EB_FIBER:
 807         case E1000_DEV_ID_82546EB_QUAD_COPPER:
 808                 return true;
 809         default:
 810                 return false;
 811         }
 812 }
 813
 814 static netdev_features_t e1000_fix_features(struct net_device *netdev,
 815         netdev_features_t features)
 816 {
 817         /* Since there is no support for separate Rx/Tx vlan accel
 818          * enable/disable make sure Tx flag is always in same state as Rx.
 819          */
 820         if (features & NETIF_F_HW_VLAN_CTAG_RX)
 821                 features |= NETIF_F_HW_VLAN_CTAG_TX;
 822         else
 823                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
 824
 825         return features;
 826 }
 827
 828 static int e1000_set_features(struct net_device *netdev,
 829         netdev_features_t features)
 830 {
 831         struct e1000_adapter *adapter = netdev_priv(netdev);
 832         netdev_features_t changed = features ^ netdev->features;
 833
 834         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
 835                 e1000_vlan_mode(netdev, features);
 836
 837         if (!(changed & (NETIF_F_RXCSUM | NETIF_F_RXALL)))
 838                 return 0;
 839
 840         netdev->features = features;
 841         adapter->rx_csum = !!(features & NETIF_F_RXCSUM);
 842
 843         if (netif_running(netdev))
 844                 e1000_reinit_locked(adapter);
 845         else
 846                 e1000_reset(adapter);
 847
 848         return 0;
 849 }
 850
 851 static const struct net_device_ops e1000_netdev_ops = {
 852         .ndo_open               = e1000_open,
 853         .ndo_stop               = e1000_close,
 854         .ndo_start_xmit         = e1000_xmit_frame,
 855         .ndo_set_rx_mode        = e1000_set_rx_mode,
 856         .ndo_set_mac_address    = e1000_set_mac,
 857         .ndo_tx_timeout         = e1000_tx_timeout,
 858         .ndo_change_mtu         = e1000_change_mtu,
 859         .ndo_do_ioctl           = e1000_ioctl,
 860         .ndo_validate_addr      = eth_validate_addr,
 861         .ndo_vlan_rx_add_vid    = e1000_vlan_rx_add_vid,
 862         .ndo_vlan_rx_kill_vid   = e1000_vlan_rx_kill_vid,
 863 #ifdef CONFIG_NET_POLL_CONTROLLER
 864         .ndo_poll_controller    = e1000_netpoll,
 865 #endif
 866         .ndo_fix_features       = e1000_fix_features,
 867         .ndo_set_features       = e1000_set_features,
 868 };
 869
 870 /**
 871  * e1000_init_hw_struct - initialize members of hw struct
 872  * @adapter: board private struct
 873  * @hw: structure used by e1000_hw.c
 874  *
 875  * Factors out initialization of the e1000_hw struct to its own function
 876  * that can be called very early at init (just after struct allocation).
 877  * Fields are initialized based on PCI device information and
 878  * OS network device settings (MTU size).
 879  * Returns negative error codes if MAC type setup fails.
 880  */
 881 static int e1000_init_hw_struct(struct e1000_adapter *adapter,
 882                                 struct e1000_hw *hw)
 883 {
 884         struct pci_dev *pdev = adapter->pdev;
 885
 886         /* PCI config space info */
 887         hw->vendor_id = pdev->vendor;
 888         hw->device_id = pdev->device;
 889         hw->subsystem_vendor_id = pdev->subsystem_vendor;
 890         hw->subsystem_id = pdev->subsystem_device;
 891         hw->revision_id = pdev->revision;
 892
 893         pci_read_config_word(pdev, PCI_COMMAND, &hw->pci_cmd_word);
 894
 895         hw->max_frame_size = adapter->netdev->mtu +
 896                              ENET_HEADER_SIZE + ETHERNET_FCS_SIZE;
 897         hw->min_frame_size = MINIMUM_ETHERNET_FRAME_SIZE;
 898
 899         /* identify the MAC */
 900         if (e1000_set_mac_type(hw)) {
 901                 e_err(probe, "Unknown MAC Type\n");
 902                 return -EIO;
 903         }
 904
 905         switch (hw->mac_type) {
 906         default:
 907                 break;
 908         case e1000_82541:
 909         case e1000_82547:
 910         case e1000_82541_rev_2:
 911         case e1000_82547_rev_2:
 912                 hw->phy_init_script = 1;
 913                 break;
 914         }
 915
 916         e1000_set_media_type(hw);
 917         e1000_get_bus_info(hw);
 918
 919         hw->wait_autoneg_complete = false;
 920         hw->tbi_compatibility_en = true;
 921         hw->adaptive_ifs = true;
 922
 923         /* Copper options */
 924
 925         if (hw->media_type == e1000_media_type_copper) {
 926                 hw->mdix = AUTO_ALL_MODES;
 927                 hw->disable_polarity_correction = false;
 928                 hw->master_slave = E1000_MASTER_SLAVE;
 929         }
 930
 931         return 0;
 932 }
 933
 934 /**
 935  * e1000_probe - Device Initialization Routine
 936  * @pdev: PCI device information struct
 937  * @ent: entry in e1000_pci_tbl
 938  *
 939  * Returns 0 on success, negative on failure
 940  *
 941  * e1000_probe initializes an adapter identified by a pci_dev structure.
 942  * The OS initialization, configuring of the adapter private structure,
 943  * and a hardware reset occur.
 944  **/
 945 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 946 {
 947         struct net_device *netdev;
 948         struct e1000_adapter *adapter;
 949         struct e1000_hw *hw;
 950
 951         static int cards_found;
 952         static int global_quad_port_a; /* global ksp3 port a indication */
 953         int i, err, pci_using_dac;
 954         u16 eeprom_data = 0;
 955         u16 tmp = 0;
 956         u16 eeprom_apme_mask = E1000_EEPROM_APME;
 957         int bars, need_ioport;
 958
 959         /* do not allocate ioport bars when not needed */
 960         need_ioport = e1000_is_need_ioport(pdev);
 961         if (need_ioport) {
 962                 bars = pci_select_bars(pdev, IORESOURCE_MEM | IORESOURCE_IO);
 963                 err = pci_enable_device(pdev);
 964         } else {
 965                 bars = pci_select_bars(pdev, IORESOURCE_MEM);
 966                 err = pci_enable_device_mem(pdev);
 967         }
 968         if (err)
 969                 return err;
 970
 971         err = pci_request_selected_regions(pdev, bars, e1000_driver_name);
 972         if (err)
 973                 goto err_pci_reg;
 974
 975         pci_set_master(pdev);
 976         err = pci_save_state(pdev);
 977         if (err)
 978                 goto err_alloc_etherdev;
 979
 980         err = -ENOMEM;
 981         netdev = alloc_etherdev(sizeof(struct e1000_adapter));
 982         if (!netdev)
 983                 goto err_alloc_etherdev;
 984
 985         SET_NETDEV_DEV(netdev, &pdev->dev);
 986
 987         pci_set_drvdata(pdev, netdev);
 988         adapter = netdev_priv(netdev);
 989         adapter->netdev = netdev;
 990         adapter->pdev = pdev;
 991         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
 992         adapter->bars = bars;
 993         adapter->need_ioport = need_ioport;
 994
 995         hw = &adapter->hw;
 996         hw->back = adapter;
 997
 998         err = -EIO;
 999         hw->hw_addr = pci_ioremap_bar(pdev, BAR_0);
1000         if (!hw->hw_addr)
1001                 goto err_ioremap;
1002
1003         if (adapter->need_ioport) {
1004                 for (i = BAR_1; i <= BAR_5; i++) {
1005                         if (pci_resource_len(pdev, i) == 0)
1006                                 continue;
1007                         if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
1008                                 hw->io_base = pci_resource_start(pdev, i);
1009                                 break;
1010                         }
1011                 }
1012         }
1013
1014         /* make ready for any if (hw->...) below */
1015         err = e1000_init_hw_struct(adapter, hw);
1016         if (err)
1017                 goto err_sw_init;
1018
1019         /* there is a workaround being applied below that limits
1020          * 64-bit DMA addresses to 64-bit hardware.  There are some
1021          * 32-bit adapters that Tx hang when given 64-bit DMA addresses
1022          */
1023         pci_using_dac = 0;
1024         if ((hw->bus_type == e1000_bus_type_pcix) &&
1025             !dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
1026                 pci_using_dac = 1;
1027         } else {
1028                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1029                 if (err) {
1030                         pr_err("No usable DMA config, aborting\n");
1031                         goto err_dma;
1032                 }
1033         }
1034
1035         netdev->netdev_ops = &e1000_netdev_ops;
1036         e1000_set_ethtool_ops(netdev);
1037         netdev->watchdog_timeo = 5 * HZ;
1038         netif_napi_add(netdev, &adapter->napi, e1000_clean, 64);
1039
1040         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1041
1042         adapter->bd_number = cards_found;
1043
1044         /* setup the private structure */
1045
1046         err = e1000_sw_init(adapter);
1047         if (err)
1048                 goto err_sw_init;
1049
1050         err = -EIO;
1051         if (hw->mac_type == e1000_ce4100) {
1052                 hw->ce4100_gbe_mdio_base_virt =
1053                                         ioremap(pci_resource_start(pdev, BAR_1),
1054                                                 pci_resource_len(pdev, BAR_1));
1055
1056                 if (!hw->ce4100_gbe_mdio_base_virt)
1057                         goto err_mdio_ioremap;
1058         }
1059
1060         if (hw->mac_type >= e1000_82543) {
1061                 netdev->hw_features = NETIF_F_SG |
1062                                    NETIF_F_HW_CSUM |
1063                                    NETIF_F_HW_VLAN_CTAG_RX;
1064                 netdev->features = NETIF_F_HW_VLAN_CTAG_TX |
1065                                    NETIF_F_HW_VLAN_CTAG_FILTER;
1066         }
1067
1068         if ((hw->mac_type >= e1000_82544) &&
1069            (hw->mac_type != e1000_82547))
1070                 netdev->hw_features |= NETIF_F_TSO;
1071
1072         netdev->priv_flags |= IFF_SUPP_NOFCS;
1073
1074         netdev->features |= netdev->hw_features;
1075         netdev->hw_features |= (NETIF_F_RXCSUM |
1076                                 NETIF_F_RXALL |
1077                                 NETIF_F_RXFCS);
1078
1079         if (pci_using_dac) {
1080                 netdev->features |= NETIF_F_HIGHDMA;
1081                 netdev->vlan_features |= NETIF_F_HIGHDMA;
1082         }
1083
1084         netdev->vlan_features |= (NETIF_F_TSO |
1085                                   NETIF_F_HW_CSUM |
1086                                   NETIF_F_SG);
1087
1088         /* Do not set IFF_UNICAST_FLT for VMWare's 82545EM */
1089         if (hw->device_id != E1000_DEV_ID_82545EM_COPPER ||
1090             hw->subsystem_vendor_id != PCI_VENDOR_ID_VMWARE)
1091                 netdev->priv_flags |= IFF_UNICAST_FLT;
1092
1093         /* MTU range: 46 - 16110 */
1094         netdev->min_mtu = ETH_ZLEN - ETH_HLEN;
1095         netdev->max_mtu = MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN);
1096
1097         adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
1098
1099         /* initialize eeprom parameters */
1100         if (e1000_init_eeprom_params(hw)) {
1101                 e_err(probe, "EEPROM initialization failed\n");
1102                 goto err_eeprom;
1103         }
1104
1105         /* before reading the EEPROM, reset the controller to
1106          * put the device in a known good starting state
1107          */
1108
1109         e1000_reset_hw(hw);
1110
1111         /* make sure the EEPROM is good */
1112         if (e1000_validate_eeprom_checksum(hw) < 0) {
1113                 e_err(probe, "The EEPROM Checksum Is Not Valid\n");
1114                 e1000_dump_eeprom(adapter);
1115                 /* set MAC address to all zeroes to invalidate and temporary
1116                  * disable this device for the user. This blocks regular
1117                  * traffic while still permitting ethtool ioctls from reaching
1118                  * the hardware as well as allowing the user to run the
1119                  * interface after manually setting a hw addr using
1120                  * `ip set address`
1121                  */
1122                 memset(hw->mac_addr, 0, netdev->addr_len);
1123         } else {
1124                 /* copy the MAC address out of the EEPROM */
1125                 if (e1000_read_mac_addr(hw))
1126                         e_err(probe, "EEPROM Read Error\n");
1127         }
1128         /* don't block initialization here due to bad MAC address */
1129         memcpy(netdev->dev_addr, hw->mac_addr, netdev->addr_len);
1130
1131         if (!is_valid_ether_addr(netdev->dev_addr))
1132                 e_err(probe, "Invalid MAC Address\n");
1133
1134
1135         INIT_DELAYED_WORK(&adapter->watchdog_task, e1000_watchdog);
1136         INIT_DELAYED_WORK(&adapter->fifo_stall_task,
1137                           e1000_82547_tx_fifo_stall_task);
1138         INIT_DELAYED_WORK(&adapter->phy_info_task, e1000_update_phy_info_task);
1139         INIT_WORK(&adapter->reset_task, e1000_reset_task);
1140
1141         e1000_check_options(adapter);
1142
1143         /* Initial Wake on LAN setting
1144          * If APM wake is enabled in the EEPROM,
1145          * enable the ACPI Magic Packet filter
1146          */
1147
1148         switch (hw->mac_type) {
1149         case e1000_82542_rev2_0:
1150         case e1000_82542_rev2_1:
1151         case e1000_82543:
1152                 break;
1153         case e1000_82544:
1154                 e1000_read_eeprom(hw,
1155                         EEPROM_INIT_CONTROL2_REG, 1, &eeprom_data);
1156                 eeprom_apme_mask = E1000_EEPROM_82544_APM;
1157                 break;
1158         case e1000_82546:
1159         case e1000_82546_rev_3:
1160                 if (er32(STATUS) & E1000_STATUS_FUNC_1) {
1161                         e1000_read_eeprom(hw,
1162                                 EEPROM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1163                         break;
1164                 }
1165                 /* Fall Through */
1166         default:
1167                 e1000_read_eeprom(hw,
1168                         EEPROM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1169                 break;
1170         }
1171         if (eeprom_data & eeprom_apme_mask)
1172                 adapter->eeprom_wol |= E1000_WUFC_MAG;
1173
1174         /* now that we have the eeprom settings, apply the special cases
1175          * where the eeprom may be wrong or the board simply won't support
1176          * wake on lan on a particular port
1177          */
1178         switch (pdev->device) {
1179         case E1000_DEV_ID_82546GB_PCIE:
1180                 adapter->eeprom_wol = 0;
1181                 break;
1182         case E1000_DEV_ID_82546EB_FIBER:
1183         case E1000_DEV_ID_82546GB_FIBER:
1184                 /* Wake events only supported on port A for dual fiber
1185                  * regardless of eeprom setting
1186                  */
1187                 if (er32(STATUS) & E1000_STATUS_FUNC_1)
1188                         adapter->eeprom_wol = 0;
1189                 break;
1190         case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
1191                 /* if quad port adapter, disable WoL on all but port A */
1192                 if (global_quad_port_a != 0)
1193                         adapter->eeprom_wol = 0;
1194                 else
1195                         adapter->quad_port_a = true;
1196                 /* Reset for multiple quad port adapters */
1197                 if (++global_quad_port_a == 4)
1198                         global_quad_port_a = 0;
1199                 break;
1200         }
1201
1202         /* initialize the wol settings based on the eeprom settings */
1203         adapter->wol = adapter->eeprom_wol;
1204         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1205
1206         /* Auto detect PHY address */
1207         if (hw->mac_type == e1000_ce4100) {
1208                 for (i = 0; i < 32; i++) {
1209                         hw->phy_addr = i;
1210                         e1000_read_phy_reg(hw, PHY_ID2, &tmp);
1211
1212                         if (tmp != 0 && tmp != 0xFF)
1213                                 break;
1214                 }
1215
1216                 if (i >= 32)
1217                         goto err_eeprom;
1218         }
1219
1220         /* reset the hardware with the new settings */
1221         e1000_reset(adapter);
1222
1223         strcpy(netdev->name, "eth%d");
1224         err = register_netdev(netdev);
1225         if (err)
1226                 goto err_register;
1227
1228         e1000_vlan_filter_on_off(adapter, false);
1229
1230         /* print bus type/speed/width info */
1231         e_info(probe, "(PCI%s:%dMHz:%d-bit) %pM\n",
1232                ((hw->bus_type == e1000_bus_type_pcix) ? "-X" : ""),
1233                ((hw->bus_speed == e1000_bus_speed_133) ? 133 :
1234                 (hw->bus_speed == e1000_bus_speed_120) ? 120 :
1235                 (hw->bus_speed == e1000_bus_speed_100) ? 100 :
1236                 (hw->bus_speed == e1000_bus_speed_66) ? 66 : 33),
1237                ((hw->bus_width == e1000_bus_width_64) ? 64 : 32),
1238                netdev->dev_addr);
1239
1240         /* carrier off reporting is important to ethtool even BEFORE open */
1241         netif_carrier_off(netdev);
1242
1243         e_info(probe, "Intel(R) PRO/1000 Network Connection\n");
1244
1245         cards_found++;
1246         return 0;
1247
1248 err_register:
1249 err_eeprom:
1250         e1000_phy_hw_reset(hw);
1251
1252         if (hw->flash_address)
1253                 iounmap(hw->flash_address);
1254         kfree(adapter->tx_ring);
1255         kfree(adapter->rx_ring);
1256 err_dma:
1257 err_sw_init:
1258 err_mdio_ioremap:
1259         iounmap(hw->ce4100_gbe_mdio_base_virt);
1260         iounmap(hw->hw_addr);
1261 err_ioremap:
1262         free_netdev(netdev);
1263 err_alloc_etherdev:
1264         pci_release_selected_regions(pdev, bars);
1265 err_pci_reg:
1266         pci_disable_device(pdev);
1267         return err;
1268 }
1269
1270 /**
1271  * e1000_remove - Device Removal Routine
1272  * @pdev: PCI device information struct
1273  *
1274  * e1000_remove is called by the PCI subsystem to alert the driver
1275  * that it should release a PCI device. That could be caused by a
1276  * Hot-Plug event, or because the driver is going to be removed from
1277  * memory.
1278  **/
1279 static void e1000_remove(struct pci_dev *pdev)
1280 {
1281         struct net_device *netdev = pci_get_drvdata(pdev);
1282         struct e1000_adapter *adapter = netdev_priv(netdev);
1283         struct e1000_hw *hw = &adapter->hw;
1284
1285         e1000_down_and_stop(adapter);
1286         e1000_release_manageability(adapter);
1287
1288         unregister_netdev(netdev);
1289
1290         e1000_phy_hw_reset(hw);
1291
1292         kfree(adapter->tx_ring);
1293         kfree(adapter->rx_ring);
1294
1295         if (hw->mac_type == e1000_ce4100)
1296                 iounmap(hw->ce4100_gbe_mdio_base_virt);
1297         iounmap(hw->hw_addr);
1298         if (hw->flash_address)
1299                 iounmap(hw->flash_address);
1300         pci_release_selected_regions(pdev, adapter->bars);
1301
1302         free_netdev(netdev);
1303
1304         pci_disable_device(pdev);
1305 }
1306
1307 /**
1308  * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
1309  * @adapter: board private structure to initialize
1310  *
1311  * e1000_sw_init initializes the Adapter private data structure.
1312  * e1000_init_hw_struct MUST be called before this function
1313  **/
1314 static int e1000_sw_init(struct e1000_adapter *adapter)
1315 {
1316         adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1317
1318         adapter->num_tx_queues = 1;
1319         adapter->num_rx_queues = 1;
1320
1321         if (e1000_alloc_queues(adapter)) {
1322                 e_err(probe, "Unable to allocate memory for queues\n");
1323                 return -ENOMEM;
1324         }
1325
1326         /* Explicitly disable IRQ since the NIC can be in any state. */
1327         e1000_irq_disable(adapter);
1328
1329         spin_lock_init(&adapter->stats_lock);
1330
1331         set_bit(__E1000_DOWN, &adapter->flags);
1332
1333         return 0;
1334 }
1335
1336 /**
1337  * e1000_alloc_queues - Allocate memory for all rings
1338  * @adapter: board private structure to initialize
1339  *
1340  * We allocate one ring per queue at run-time since we don't know the
1341  * number of queues at compile-time.
1342  **/
1343 static int e1000_alloc_queues(struct e1000_adapter *adapter)
1344 {
1345         adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1346                                    sizeof(struct e1000_tx_ring), GFP_KERNEL);
1347         if (!adapter->tx_ring)
1348                 return -ENOMEM;
1349
1350         adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1351                                    sizeof(struct e1000_rx_ring), GFP_KERNEL);
1352         if (!adapter->rx_ring) {
1353                 kfree(adapter->tx_ring);
1354                 return -ENOMEM;
1355         }
1356
1357         return E1000_SUCCESS;
1358 }
1359
1360 /**
1361  * e1000_open - Called when a network interface is made active
1362  * @netdev: network interface device structure
1363  *
1364  * Returns 0 on success, negative value on failure
1365  *
1366  * The open entry point is called when a network interface is made
1367  * active by the system (IFF_UP).  At this point all resources needed
1368  * for transmit and receive operations are allocated, the interrupt
1369  * handler is registered with the OS, the watchdog task is started,
1370  * and the stack is notified that the interface is ready.
1371  **/
1372 int e1000_open(struct net_device *netdev)
1373 {
1374         struct e1000_adapter *adapter = netdev_priv(netdev);
1375         struct e1000_hw *hw = &adapter->hw;
1376         int err;
1377
1378         /* disallow open during test */
1379         if (test_bit(__E1000_TESTING, &adapter->flags))
1380                 return -EBUSY;
1381
1382         netif_carrier_off(netdev);
1383
1384         /* allocate transmit descriptors */
1385         err = e1000_setup_all_tx_resources(adapter);
1386         if (err)
1387                 goto err_setup_tx;
1388
1389         /* allocate receive descriptors */
1390         err = e1000_setup_all_rx_resources(adapter);
1391         if (err)
1392                 goto err_setup_rx;
1393
1394         e1000_power_up_phy(adapter);
1395
1396         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
1397         if ((hw->mng_cookie.status &
1398                           E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT)) {
1399                 e1000_update_mng_vlan(adapter);
1400         }
1401
1402         /* before we allocate an interrupt, we must be ready to handle it.
1403          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1404          * as soon as we call pci_request_irq, so we have to setup our
1405          * clean_rx handler before we do so.
1406          */
1407         e1000_configure(adapter);
1408
1409         err = e1000_request_irq(adapter);
1410         if (err)
1411                 goto err_req_irq;
1412
1413         /* From here on the code is the same as e1000_up() */
1414         clear_bit(__E1000_DOWN, &adapter->flags);
1415
1416         napi_enable(&adapter->napi);
1417
1418         e1000_irq_enable(adapter);
1419
1420         netif_start_queue(netdev);
1421
1422         /* fire a link status change interrupt to start the watchdog */
1423         ew32(ICS, E1000_ICS_LSC);
1424
1425         return E1000_SUCCESS;
1426
1427 err_req_irq:
1428         e1000_power_down_phy(adapter);
1429         e1000_free_all_rx_resources(adapter);
1430 err_setup_rx:
1431         e1000_free_all_tx_resources(adapter);
1432 err_setup_tx:
1433         e1000_reset(adapter);
1434
1435         return err;
1436 }
1437
1438 /**
1439  * e1000_close - Disables a network interface
1440  * @netdev: network interface device structure
1441  *
1442  * Returns 0, this is not allowed to fail
1443  *
1444  * The close entry point is called when an interface is de-activated
1445  * by the OS.  The hardware is still under the drivers control, but
1446  * needs to be disabled.  A global MAC reset is issued to stop the
1447  * hardware, and all transmit and receive resources are freed.
1448  **/
1449 int e1000_close(struct net_device *netdev)
1450 {
1451         struct e1000_adapter *adapter = netdev_priv(netdev);
1452         struct e1000_hw *hw = &adapter->hw;
1453         int count = E1000_CHECK_RESET_COUNT;
1454
1455         while (test_bit(__E1000_RESETTING, &adapter->flags) && count--)
1456                 usleep_range(10000, 20000);
1457
1458         WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
1459         e1000_down(adapter);
1460         e1000_power_down_phy(adapter);
1461         e1000_free_irq(adapter);
1462
1463         e1000_free_all_tx_resources(adapter);
1464         e1000_free_all_rx_resources(adapter);
1465
1466         /* kill manageability vlan ID if supported, but not if a vlan with
1467          * the same ID is registered on the host OS (let 8021q kill it)
1468          */
1469         if ((hw->mng_cookie.status &
1470              E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
1471             !test_bit(adapter->mng_vlan_id, adapter->active_vlans)) {
1472                 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
1473                                        adapter->mng_vlan_id);
1474         }
1475
1476         return 0;
1477 }
1478
1479 /**
1480  * e1000_check_64k_bound - check that memory doesn't cross 64kB boundary
1481  * @adapter: address of board private structure
1482  * @start: address of beginning of memory
1483  * @len: length of memory
1484  **/
1485 static bool e1000_check_64k_bound(struct e1000_adapter *adapter, void *start,
1486                                   unsigned long len)
1487 {
1488         struct e1000_hw *hw = &adapter->hw;
1489         unsigned long begin = (unsigned long)start;
1490         unsigned long end = begin + len;
1491
1492         /* First rev 82545 and 82546 need to not allow any memory
1493          * write location to cross 64k boundary due to errata 23
1494          */
1495         if (hw->mac_type == e1000_82545 ||
1496             hw->mac_type == e1000_ce4100 ||
1497             hw->mac_type == e1000_82546) {
1498                 return ((begin ^ (end - 1)) >> 16) != 0 ? false : true;
1499         }
1500
1501         return true;
1502 }
1503
1504 /**
1505  * e1000_setup_tx_resources - allocate Tx resources (Descriptors)
1506  * @adapter: board private structure
1507  * @txdr:    tx descriptor ring (for a specific queue) to setup
1508  *
1509  * Return 0 on success, negative on failure
1510  **/
1511 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
1512                                     struct e1000_tx_ring *txdr)
1513 {
1514         struct pci_dev *pdev = adapter->pdev;
1515         int size;
1516
1517         size = sizeof(struct e1000_tx_buffer) * txdr->count;
1518         txdr->buffer_info = vzalloc(size);
1519         if (!txdr->buffer_info)
1520                 return -ENOMEM;
1521
1522         /* round up to nearest 4K */
1523
1524         txdr->size = txdr->count * sizeof(struct e1000_tx_desc);
1525         txdr->size = ALIGN(txdr->size, 4096);
1526
1527         txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size, &txdr->dma,
1528                                         GFP_KERNEL);
1529         if (!txdr->desc) {
1530 setup_tx_desc_die:
1531                 vfree(txdr->buffer_info);
1532                 return -ENOMEM;
1533         }
1534
1535         /* Fix for errata 23, can't cross 64kB boundary */
1536         if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1537                 void *olddesc = txdr->desc;
1538                 dma_addr_t olddma = txdr->dma;
1539                 e_err(tx_err, "txdr align check failed: %u bytes at %p\n",
1540                       txdr->size, txdr->desc);
1541                 /* Try again, without freeing the previous */
1542                 txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size,
1543                                                 &txdr->dma, GFP_KERNEL);
1544                 /* Failed allocation, critical failure */
1545                 if (!txdr->desc) {
1546                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1547                                           olddma);
1548                         goto setup_tx_desc_die;
1549                 }
1550
1551                 if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1552                         /* give up */
1553                         dma_free_coherent(&pdev->dev, txdr->size, txdr->desc,
1554                                           txdr->dma);
1555                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1556                                           olddma);
1557                         e_err(probe, "Unable to allocate aligned memory "
1558                               "for the transmit descriptor ring\n");
1559                         vfree(txdr->buffer_info);
1560                         return -ENOMEM;
1561                 } else {
1562                         /* Free old allocation, new allocation was successful */
1563                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1564                                           olddma);
1565                 }
1566         }
1567         memset(txdr->desc, 0, txdr->size);
1568
1569         txdr->next_to_use = 0;
1570         txdr->next_to_clean = 0;
1571
1572         return 0;
1573 }
1574
1575 /**
1576  * e1000_setup_all_tx_resources - wrapper to allocate Tx resources
1577  *                                (Descriptors) for all queues
1578  * @adapter: board private structure
1579  *
1580  * Return 0 on success, negative on failure
1581  **/
1582 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter)
1583 {
1584         int i, err = 0;
1585
1586         for (i = 0; i < adapter->num_tx_queues; i++) {
1587                 err = e1000_setup_tx_resources(adapter, &adapter->tx_ring[i]);
1588                 if (err) {
1589                         e_err(probe, "Allocation for Tx Queue %u failed\n", i);
1590                         for (i-- ; i >= 0; i--)
1591                                 e1000_free_tx_resources(adapter,
1592                                                         &adapter->tx_ring[i]);
1593                         break;
1594                 }
1595         }
1596
1597         return err;
1598 }
1599
1600 /**
1601  * e1000_configure_tx - Configure 8254x Transmit Unit after Reset
1602  * @adapter: board private structure
1603  *
1604  * Configure the Tx unit of the MAC after a reset.
1605  **/
1606 static void e1000_configure_tx(struct e1000_adapter *adapter)
1607 {
1608         u64 tdba;
1609         struct e1000_hw *hw = &adapter->hw;
1610         u32 tdlen, tctl, tipg;
1611         u32 ipgr1, ipgr2;
1612
1613         /* Setup the HW Tx Head and Tail descriptor pointers */
1614
1615         switch (adapter->num_tx_queues) {
1616         case 1:
1617         default:
1618                 tdba = adapter->tx_ring[0].dma;
1619                 tdlen = adapter->tx_ring[0].count *
1620                         sizeof(struct e1000_tx_desc);
1621                 ew32(TDLEN, tdlen);
1622                 ew32(TDBAH, (tdba >> 32));
1623                 ew32(TDBAL, (tdba & 0x00000000ffffffffULL));
1624                 ew32(TDT, 0);
1625                 ew32(TDH, 0);
1626                 adapter->tx_ring[0].tdh = ((hw->mac_type >= e1000_82543) ?
1627                                            E1000_TDH : E1000_82542_TDH);
1628                 adapter->tx_ring[0].tdt = ((hw->mac_type >= e1000_82543) ?
1629                                            E1000_TDT : E1000_82542_TDT);
1630                 break;
1631         }
1632
1633         /* Set the default values for the Tx Inter Packet Gap timer */
1634         if ((hw->media_type == e1000_media_type_fiber ||
1635              hw->media_type == e1000_media_type_internal_serdes))
1636                 tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1637         else
1638                 tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1639
1640         switch (hw->mac_type) {
1641         case e1000_82542_rev2_0:
1642         case e1000_82542_rev2_1:
1643                 tipg = DEFAULT_82542_TIPG_IPGT;
1644                 ipgr1 = DEFAULT_82542_TIPG_IPGR1;
1645                 ipgr2 = DEFAULT_82542_TIPG_IPGR2;
1646                 break;
1647         default:
1648                 ipgr1 = DEFAULT_82543_TIPG_IPGR1;
1649                 ipgr2 = DEFAULT_82543_TIPG_IPGR2;
1650                 break;
1651         }
1652         tipg |= ipgr1 << E1000_TIPG_IPGR1_SHIFT;
1653         tipg |= ipgr2 << E1000_TIPG_IPGR2_SHIFT;
1654         ew32(TIPG, tipg);
1655
1656         /* Set the Tx Interrupt Delay register */
1657
1658         ew32(TIDV, adapter->tx_int_delay);
1659         if (hw->mac_type >= e1000_82540)
1660                 ew32(TADV, adapter->tx_abs_int_delay);
1661
1662         /* Program the Transmit Control Register */
1663
1664         tctl = er32(TCTL);
1665         tctl &= ~E1000_TCTL_CT;
1666         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
1667                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1668
1669         e1000_config_collision_dist(hw);
1670
1671         /* Setup Transmit Descriptor Settings for eop descriptor */
1672         adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
1673
1674         /* only set IDE if we are delaying interrupts using the timers */
1675         if (adapter->tx_int_delay)
1676                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
1677
1678         if (hw->mac_type < e1000_82543)
1679                 adapter->txd_cmd |= E1000_TXD_CMD_RPS;
1680         else
1681                 adapter->txd_cmd |= E1000_TXD_CMD_RS;
1682
1683         /* Cache if we're 82544 running in PCI-X because we'll
1684          * need this to apply a workaround later in the send path.
1685          */
1686         if (hw->mac_type == e1000_82544 &&
1687             hw->bus_type == e1000_bus_type_pcix)
1688                 adapter->pcix_82544 = true;
1689
1690         ew32(TCTL, tctl);
1691
1692 }
1693
1694 /**
1695  * e1000_setup_rx_resources - allocate Rx resources (Descriptors)
1696  * @adapter: board private structure
1697  * @rxdr:    rx descriptor ring (for a specific queue) to setup
1698  *
1699  * Returns 0 on success, negative on failure
1700  **/
1701 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
1702                                     struct e1000_rx_ring *rxdr)
1703 {
1704         struct pci_dev *pdev = adapter->pdev;
1705         int size, desc_len;
1706
1707         size = sizeof(struct e1000_rx_buffer) * rxdr->count;
1708         rxdr->buffer_info = vzalloc(size);
1709         if (!rxdr->buffer_info)
1710                 return -ENOMEM;
1711
1712         desc_len = sizeof(struct e1000_rx_desc);
1713
1714         /* Round up to nearest 4K */
1715
1716         rxdr->size = rxdr->count * desc_len;
1717         rxdr->size = ALIGN(rxdr->size, 4096);
1718
1719         rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size, &rxdr->dma,
1720                                         GFP_KERNEL);
1721         if (!rxdr->desc) {
1722 setup_rx_desc_die:
1723                 vfree(rxdr->buffer_info);
1724                 return -ENOMEM;
1725         }
1726
1727         /* Fix for errata 23, can't cross 64kB boundary */
1728         if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1729                 void *olddesc = rxdr->desc;
1730                 dma_addr_t olddma = rxdr->dma;
1731                 e_err(rx_err, "rxdr align check failed: %u bytes at %p\n",
1732                       rxdr->size, rxdr->desc);
1733                 /* Try again, without freeing the previous */
1734                 rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size,
1735                                                 &rxdr->dma, GFP_KERNEL);
1736                 /* Failed allocation, critical failure */
1737                 if (!rxdr->desc) {
1738                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1739                                           olddma);
1740                         goto setup_rx_desc_die;
1741                 }
1742
1743                 if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1744                         /* give up */
1745                         dma_free_coherent(&pdev->dev, rxdr->size, rxdr->desc,
1746                                           rxdr->dma);
1747                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1748                                           olddma);
1749                         e_err(probe, "Unable to allocate aligned memory for "
1750                               "the Rx descriptor ring\n");
1751                         goto setup_rx_desc_die;
1752                 } else {
1753                         /* Free old allocation, new allocation was successful */
1754                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1755                                           olddma);
1756                 }
1757         }
1758         memset(rxdr->desc, 0, rxdr->size);
1759
1760         rxdr->next_to_clean = 0;
1761         rxdr->next_to_use = 0;
1762         rxdr->rx_skb_top = NULL;
1763
1764         return 0;
1765 }
1766
1767 /**
1768  * e1000_setup_all_rx_resources - wrapper to allocate Rx resources
1769  *                                (Descriptors) for all queues
1770  * @adapter: board private structure
1771  *
1772  * Return 0 on success, negative on failure
1773  **/
1774 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter)
1775 {
1776         int i, err = 0;
1777
1778         for (i = 0; i < adapter->num_rx_queues; i++) {
1779                 err = e1000_setup_rx_resources(adapter, &adapter->rx_ring[i]);
1780                 if (err) {
1781                         e_err(probe, "Allocation for Rx Queue %u failed\n", i);
1782                         for (i-- ; i >= 0; i--)
1783                                 e1000_free_rx_resources(adapter,
1784                                                         &adapter->rx_ring[i]);
1785                         break;
1786                 }
1787         }
1788
1789         return err;
1790 }
1791
1792 /**
1793  * e1000_setup_rctl - configure the receive control registers
1794  * @adapter: Board private structure
1795  **/
1796 static void e1000_setup_rctl(struct e1000_adapter *adapter)
1797 {
1798         struct e1000_hw *hw = &adapter->hw;
1799         u32 rctl;
1800
1801         rctl = er32(RCTL);
1802
1803         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
1804
1805         rctl |= E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
1806                 E1000_RCTL_RDMTS_HALF |
1807                 (hw->mc_filter_type << E1000_RCTL_MO_SHIFT);
1808
1809         if (hw->tbi_compatibility_on == 1)
1810                 rctl |= E1000_RCTL_SBP;
1811         else
1812                 rctl &= ~E1000_RCTL_SBP;
1813
1814         if (adapter->netdev->mtu <= ETH_DATA_LEN)
1815                 rctl &= ~E1000_RCTL_LPE;
1816         else
1817                 rctl |= E1000_RCTL_LPE;
1818
1819         /* Setup buffer sizes */
1820         rctl &= ~E1000_RCTL_SZ_4096;
1821         rctl |= E1000_RCTL_BSEX;
1822         switch (adapter->rx_buffer_len) {
1823         case E1000_RXBUFFER_2048:
1824         default:
1825                 rctl |= E1000_RCTL_SZ_2048;
1826                 rctl &= ~E1000_RCTL_BSEX;
1827                 break;
1828         case E1000_RXBUFFER_4096:
1829                 rctl |= E1000_RCTL_SZ_4096;
1830                 break;
1831         case E1000_RXBUFFER_8192:
1832                 rctl |= E1000_RCTL_SZ_8192;
1833                 break;
1834         case E1000_RXBUFFER_16384:
1835                 rctl |= E1000_RCTL_SZ_16384;
1836                 break;
1837         }
1838
1839         /* This is useful for sniffing bad packets. */
1840         if (adapter->netdev->features & NETIF_F_RXALL) {
1841                 /* UPE and MPE will be handled by normal PROMISC logic
1842                  * in e1000e_set_rx_mode
1843                  */
1844                 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
1845                          E1000_RCTL_BAM | /* RX All Bcast Pkts */
1846                          E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
1847
1848                 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
1849                           E1000_RCTL_DPF | /* Allow filtered pause */
1850                           E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
1851                 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
1852                  * and that breaks VLANs.
1853                  */
1854         }
1855
1856         ew32(RCTL, rctl);
1857 }
1858
1859 /**
1860  * e1000_configure_rx - Configure 8254x Receive Unit after Reset
1861  * @adapter: board private structure
1862  *
1863  * Configure the Rx unit of the MAC after a reset.
1864  **/
1865 static void e1000_configure_rx(struct e1000_adapter *adapter)
1866 {
1867         u64 rdba;
1868         struct e1000_hw *hw = &adapter->hw;
1869         u32 rdlen, rctl, rxcsum;
1870
1871         if (adapter->netdev->mtu > ETH_DATA_LEN) {
1872                 rdlen = adapter->rx_ring[0].count *
1873                         sizeof(struct e1000_rx_desc);
1874                 adapter->clean_rx = e1000_clean_jumbo_rx_irq;
1875                 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
1876         } else {
1877                 rdlen = adapter->rx_ring[0].count *
1878                         sizeof(struct e1000_rx_desc);
1879                 adapter->clean_rx = e1000_clean_rx_irq;
1880                 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
1881         }
1882
1883         /* disable receives while setting up the descriptors */
1884         rctl = er32(RCTL);
1885         ew32(RCTL, rctl & ~E1000_RCTL_EN);
1886
1887         /* set the Receive Delay Timer Register */
1888         ew32(RDTR, adapter->rx_int_delay);
1889
1890         if (hw->mac_type >= e1000_82540) {
1891                 ew32(RADV, adapter->rx_abs_int_delay);
1892                 if (adapter->itr_setting != 0)
1893                         ew32(ITR, 1000000000 / (adapter->itr * 256));
1894         }
1895
1896         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1897          * the Base and Length of the Rx Descriptor Ring
1898          */
1899         switch (adapter->num_rx_queues) {
1900         case 1:
1901         default:
1902                 rdba = adapter->rx_ring[0].dma;
1903                 ew32(RDLEN, rdlen);
1904                 ew32(RDBAH, (rdba >> 32));
1905                 ew32(RDBAL, (rdba & 0x00000000ffffffffULL));
1906                 ew32(RDT, 0);
1907                 ew32(RDH, 0);
1908                 adapter->rx_ring[0].rdh = ((hw->mac_type >= e1000_82543) ?
1909                                            E1000_RDH : E1000_82542_RDH);
1910                 adapter->rx_ring[0].rdt = ((hw->mac_type >= e1000_82543) ?
1911                                            E1000_RDT : E1000_82542_RDT);
1912                 break;
1913         }
1914
1915         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
1916         if (hw->mac_type >= e1000_82543) {
1917                 rxcsum = er32(RXCSUM);
1918                 if (adapter->rx_csum)
1919                         rxcsum |= E1000_RXCSUM_TUOFL;
1920                 else
1921                         /* don't need to clear IPPCSE as it defaults to 0 */
1922                         rxcsum &= ~E1000_RXCSUM_TUOFL;
1923                 ew32(RXCSUM, rxcsum);
1924         }
1925
1926         /* Enable Receives */
1927         ew32(RCTL, rctl | E1000_RCTL_EN);
1928 }
1929
1930 /**
1931  * e1000_free_tx_resources - Free Tx Resources per Queue
1932  * @adapter: board private structure
1933  * @tx_ring: Tx descriptor ring for a specific queue
1934  *
1935  * Free all transmit software resources
1936  **/
1937 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
1938                                     struct e1000_tx_ring *tx_ring)
1939 {
1940         struct pci_dev *pdev = adapter->pdev;
1941
1942         e1000_clean_tx_ring(adapter, tx_ring);
1943
1944         vfree(tx_ring->buffer_info);
1945         tx_ring->buffer_info = NULL;
1946
1947         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
1948                           tx_ring->dma);
1949
1950         tx_ring->desc = NULL;
1951 }
1952
1953 /**
1954  * e1000_free_all_tx_resources - Free Tx Resources for All Queues
1955  * @adapter: board private structure
1956  *
1957  * Free all transmit software resources
1958  **/
1959 void e1000_free_all_tx_resources(struct e1000_adapter *adapter)
1960 {
1961         int i;
1962
1963         for (i = 0; i < adapter->num_tx_queues; i++)
1964                 e1000_free_tx_resources(adapter, &adapter->tx_ring[i]);
1965 }
1966
1967 static void
1968 e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter,
1969                                  struct e1000_tx_buffer *buffer_info)
1970 {
1971         if (buffer_info->dma) {
1972                 if (buffer_info->mapped_as_page)
1973                         dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1974                                        buffer_info->length, DMA_TO_DEVICE);
1975                 else
1976                         dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1977                                          buffer_info->length,
1978                                          DMA_TO_DEVICE);
1979                 buffer_info->dma = 0;
1980         }
1981         if (buffer_info->skb) {
1982                 dev_kfree_skb_any(buffer_info->skb);
1983                 buffer_info->skb = NULL;
1984         }
1985         buffer_info->time_stamp = 0;
1986         /* buffer_info must be completely set up in the transmit path */
1987 }
1988
1989 /**
1990  * e1000_clean_tx_ring - Free Tx Buffers
1991  * @adapter: board private structure
1992  * @tx_ring: ring to be cleaned
1993  **/
1994 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
1995                                 struct e1000_tx_ring *tx_ring)
1996 {
1997         struct e1000_hw *hw = &adapter->hw;
1998         struct e1000_tx_buffer *buffer_info;
1999         unsigned long size;
2000         unsigned int i;
2001
2002         /* Free all the Tx ring sk_buffs */
2003
2004         for (i = 0; i < tx_ring->count; i++) {
2005                 buffer_info = &tx_ring->buffer_info[i];
2006                 e1000_unmap_and_free_tx_resource(adapter, buffer_info);
2007         }
2008
2009         netdev_reset_queue(adapter->netdev);
2010         size = sizeof(struct e1000_tx_buffer) * tx_ring->count;
2011         memset(tx_ring->buffer_info, 0, size);
2012
2013         /* Zero out the descriptor ring */
2014
2015         memset(tx_ring->desc, 0, tx_ring->size);
2016
2017         tx_ring->next_to_use = 0;
2018         tx_ring->next_to_clean = 0;
2019         tx_ring->last_tx_tso = false;
2020
2021         writel(0, hw->hw_addr + tx_ring->tdh);
2022         writel(0, hw->hw_addr + tx_ring->tdt);
2023 }
2024
2025 /**
2026  * e1000_clean_all_tx_rings - Free Tx Buffers for all queues
2027  * @adapter: board private structure
2028  **/
2029 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter)
2030 {
2031         int i;
2032
2033         for (i = 0; i < adapter->num_tx_queues; i++)
2034                 e1000_clean_tx_ring(adapter, &adapter->tx_ring[i]);
2035 }
2036
2037 /**
2038  * e1000_free_rx_resources - Free Rx Resources
2039  * @adapter: board private structure
2040  * @rx_ring: ring to clean the resources from
2041  *
2042  * Free all receive software resources
2043  **/
2044 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
2045                                     struct e1000_rx_ring *rx_ring)
2046 {
2047         struct pci_dev *pdev = adapter->pdev;
2048
2049         e1000_clean_rx_ring(adapter, rx_ring);
2050
2051         vfree(rx_ring->buffer_info);
2052         rx_ring->buffer_info = NULL;
2053
2054         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2055                           rx_ring->dma);
2056
2057         rx_ring->desc = NULL;
2058 }
2059
2060 /**
2061  * e1000_free_all_rx_resources - Free Rx Resources for All Queues
2062  * @adapter: board private structure
2063  *
2064  * Free all receive software resources
2065  **/
2066 void e1000_free_all_rx_resources(struct e1000_adapter *adapter)
2067 {
2068         int i;
2069
2070         for (i = 0; i < adapter->num_rx_queues; i++)
2071                 e1000_free_rx_resources(adapter, &adapter->rx_ring[i]);
2072 }
2073
2074 #define E1000_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
2075 static unsigned int e1000_frag_len(const struct e1000_adapter *a)
2076 {
2077         return SKB_DATA_ALIGN(a->rx_buffer_len + E1000_HEADROOM) +
2078                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2079 }
2080
2081 static void *e1000_alloc_frag(const struct e1000_adapter *a)
2082 {
2083         unsigned int len = e1000_frag_len(a);
2084         u8 *data = netdev_alloc_frag(len);
2085
2086         if (likely(data))
2087                 data += E1000_HEADROOM;
2088         return data;
2089 }
2090
2091 /**
2092  * e1000_clean_rx_ring - Free Rx Buffers per Queue
2093  * @adapter: board private structure
2094  * @rx_ring: ring to free buffers from
2095  **/
2096 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
2097                                 struct e1000_rx_ring *rx_ring)
2098 {
2099         struct e1000_hw *hw = &adapter->hw;
2100         struct e1000_rx_buffer *buffer_info;
2101         struct pci_dev *pdev = adapter->pdev;
2102         unsigned long size;
2103         unsigned int i;
2104
2105         /* Free all the Rx netfrags */
2106         for (i = 0; i < rx_ring->count; i++) {
2107                 buffer_info = &rx_ring->buffer_info[i];
2108                 if (adapter->clean_rx == e1000_clean_rx_irq) {
2109                         if (buffer_info->dma)
2110                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
2111                                                  adapter->rx_buffer_len,
2112                                                  DMA_FROM_DEVICE);
2113                         if (buffer_info->rxbuf.data) {
2114                                 skb_free_frag(buffer_info->rxbuf.data);
2115                                 buffer_info->rxbuf.data = NULL;
2116                         }
2117                 } else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) {
2118                         if (buffer_info->dma)
2119                                 dma_unmap_page(&pdev->dev, buffer_info->dma,
2120                                                adapter->rx_buffer_len,
2121                                                DMA_FROM_DEVICE);
2122                         if (buffer_info->rxbuf.page) {
2123                                 put_page(buffer_info->rxbuf.page);
2124                                 buffer_info->rxbuf.page = NULL;
2125                         }
2126                 }
2127
2128                 buffer_info->dma = 0;
2129         }
2130
2131         /* there also may be some cached data from a chained receive */
2132         napi_free_frags(&adapter->napi);
2133         rx_ring->rx_skb_top = NULL;
2134
2135         size = sizeof(struct e1000_rx_buffer) * rx_ring->count;
2136         memset(rx_ring->buffer_info, 0, size);
2137
2138         /* Zero out the descriptor ring */
2139         memset(rx_ring->desc, 0, rx_ring->size);
2140
2141         rx_ring->next_to_clean = 0;
2142         rx_ring->next_to_use = 0;
2143
2144         writel(0, hw->hw_addr + rx_ring->rdh);
2145         writel(0, hw->hw_addr + rx_ring->rdt);
2146 }
2147
2148 /**
2149  * e1000_clean_all_rx_rings - Free Rx Buffers for all queues
2150  * @adapter: board private structure
2151  **/
2152 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter)
2153 {
2154         int i;
2155
2156         for (i = 0; i < adapter->num_rx_queues; i++)
2157                 e1000_clean_rx_ring(adapter, &adapter->rx_ring[i]);
2158 }
2159
2160 /* The 82542 2.0 (revision 2) needs to have the receive unit in reset
2161  * and memory write and invalidate disabled for certain operations
2162  */
2163 static void e1000_enter_82542_rst(struct e1000_adapter *adapter)
2164 {
2165         struct e1000_hw *hw = &adapter->hw;
2166         struct net_device *netdev = adapter->netdev;
2167         u32 rctl;
2168
2169         e1000_pci_clear_mwi(hw);
2170
2171         rctl = er32(RCTL);
2172         rctl |= E1000_RCTL_RST;
2173         ew32(RCTL, rctl);
2174         E1000_WRITE_FLUSH();
2175         mdelay(5);
2176
2177         if (netif_running(netdev))
2178                 e1000_clean_all_rx_rings(adapter);
2179 }
2180
2181 static void e1000_leave_82542_rst(struct e1000_adapter *adapter)
2182 {
2183         struct e1000_hw *hw = &adapter->hw;
2184         struct net_device *netdev = adapter->netdev;
2185         u32 rctl;
2186
2187         rctl = er32(RCTL);
2188         rctl &= ~E1000_RCTL_RST;
2189         ew32(RCTL, rctl);
2190         E1000_WRITE_FLUSH();
2191         mdelay(5);
2192
2193         if (hw->pci_cmd_word & PCI_COMMAND_INVALIDATE)
2194                 e1000_pci_set_mwi(hw);
2195
2196         if (netif_running(netdev)) {
2197                 /* No need to loop, because 82542 supports only 1 queue */
2198                 struct e1000_rx_ring *ring = &adapter->rx_ring[0];
2199                 e1000_configure_rx(adapter);
2200                 adapter->alloc_rx_buf(adapter, ring, E1000_DESC_UNUSED(ring));
2201         }
2202 }
2203
2204 /**
2205  * e1000_set_mac - Change the Ethernet Address of the NIC
2206  * @netdev: network interface device structure
2207  * @p: pointer to an address structure
2208  *
2209  * Returns 0 on success, negative on failure
2210  **/
2211 static int e1000_set_mac(struct net_device *netdev, void *p)
2212 {
2213         struct e1000_adapter *adapter = netdev_priv(netdev);
2214         struct e1000_hw *hw = &adapter->hw;
2215         struct sockaddr *addr = p;
2216
2217         if (!is_valid_ether_addr(addr->sa_data))
2218                 return -EADDRNOTAVAIL;
2219
2220         /* 82542 2.0 needs to be in reset to write receive address registers */
2221
2222         if (hw->mac_type == e1000_82542_rev2_0)
2223                 e1000_enter_82542_rst(adapter);
2224
2225         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2226         memcpy(hw->mac_addr, addr->sa_data, netdev->addr_len);
2227
2228         e1000_rar_set(hw, hw->mac_addr, 0);
2229
2230         if (hw->mac_type == e1000_82542_rev2_0)
2231                 e1000_leave_82542_rst(adapter);
2232
2233         return 0;
2234 }
2235
2236 /**
2237  * e1000_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2238  * @netdev: network interface device structure
2239  *
2240  * The set_rx_mode entry point is called whenever the unicast or multicast
2241  * address lists or the network interface flags are updated. This routine is
2242  * responsible for configuring the hardware for proper unicast, multicast,
2243  * promiscuous mode, and all-multi behavior.
2244  **/
2245 static void e1000_set_rx_mode(struct net_device *netdev)
2246 {
2247         struct e1000_adapter *adapter = netdev_priv(netdev);
2248         struct e1000_hw *hw = &adapter->hw;
2249         struct netdev_hw_addr *ha;
2250         bool use_uc = false;
2251         u32 rctl;
2252         u32 hash_value;
2253         int i, rar_entries = E1000_RAR_ENTRIES;
2254         int mta_reg_count = E1000_NUM_MTA_REGISTERS;
2255         u32 *mcarray = kcalloc(mta_reg_count, sizeof(u32), GFP_ATOMIC);
2256
2257         if (!mcarray)
2258                 return;
2259
2260         /* Check for Promiscuous and All Multicast modes */
2261
2262         rctl = er32(RCTL);
2263
2264         if (netdev->flags & IFF_PROMISC) {
2265                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2266                 rctl &= ~E1000_RCTL_VFE;
2267         } else {
2268                 if (netdev->flags & IFF_ALLMULTI)
2269                         rctl |= E1000_RCTL_MPE;
2270                 else
2271                         rctl &= ~E1000_RCTL_MPE;
2272                 /* Enable VLAN filter if there is a VLAN */
2273                 if (e1000_vlan_used(adapter))
2274                         rctl |= E1000_RCTL_VFE;
2275         }
2276
2277         if (netdev_uc_count(netdev) > rar_entries - 1) {
2278                 rctl |= E1000_RCTL_UPE;
2279         } else if (!(netdev->flags & IFF_PROMISC)) {
2280                 rctl &= ~E1000_RCTL_UPE;
2281                 use_uc = true;
2282         }
2283
2284         ew32(RCTL, rctl);
2285
2286         /* 82542 2.0 needs to be in reset to write receive address registers */
2287
2288         if (hw->mac_type == e1000_82542_rev2_0)
2289                 e1000_enter_82542_rst(adapter);
2290
2291         /* load the first 14 addresses into the exact filters 1-14. Unicast
2292          * addresses take precedence to avoid disabling unicast filtering
2293          * when possible.
2294          *
2295          * RAR 0 is used for the station MAC address
2296          * if there are not 14 addresses, go ahead and clear the filters
2297          */
2298         i = 1;
2299         if (use_uc)
2300                 netdev_for_each_uc_addr(ha, netdev) {
2301                         if (i == rar_entries)
2302                                 break;
2303                         e1000_rar_set(hw, ha->addr, i++);
2304                 }
2305
2306         netdev_for_each_mc_addr(ha, netdev) {
2307                 if (i == rar_entries) {
2308                         /* load any remaining addresses into the hash table */
2309                         u32 hash_reg, hash_bit, mta;
2310                         hash_value = e1000_hash_mc_addr(hw, ha->addr);
2311                         hash_reg = (hash_value >> 5) & 0x7F;
2312                         hash_bit = hash_value & 0x1F;
2313                         mta = (1 << hash_bit);
2314                         mcarray[hash_reg] |= mta;
2315                 } else {
2316                         e1000_rar_set(hw, ha->addr, i++);
2317                 }
2318         }
2319
2320         for (; i < rar_entries; i++) {
2321                 E1000_WRITE_REG_ARRAY(hw, RA, i << 1, 0);
2322                 E1000_WRITE_FLUSH();
2323                 E1000_WRITE_REG_ARRAY(hw, RA, (i << 1) + 1, 0);
2324                 E1000_WRITE_FLUSH();
2325         }
2326
2327         /* write the hash table completely, write from bottom to avoid
2328          * both stupid write combining chipsets, and flushing each write
2329          */
2330         for (i = mta_reg_count - 1; i >= 0 ; i--) {
2331                 /* If we are on an 82544 has an errata where writing odd
2332                  * offsets overwrites the previous even offset, but writing
2333                  * backwards over the range solves the issue by always
2334                  * writing the odd offset first
2335                  */
2336                 E1000_WRITE_REG_ARRAY(hw, MTA, i, mcarray[i]);
2337         }
2338         E1000_WRITE_FLUSH();
2339
2340         if (hw->mac_type == e1000_82542_rev2_0)
2341                 e1000_leave_82542_rst(adapter);
2342
2343         kfree(mcarray);
2344 }
2345
2346 /**
2347  * e1000_update_phy_info_task - get phy info
2348  * @work: work struct contained inside adapter struct
2349  *
2350  * Need to wait a few seconds after link up to get diagnostic information from
2351  * the phy
2352  */
2353 static void e1000_update_phy_info_task(struct work_struct *work)
2354 {
2355         struct e1000_adapter *adapter = container_of(work,
2356                                                      struct e1000_adapter,
2357                                                      phy_info_task.work);
2358
2359         e1000_phy_get_info(&adapter->hw, &adapter->phy_info);
2360 }
2361
2362 /**
2363  * e1000_82547_tx_fifo_stall_task - task to complete work
2364  * @work: work struct contained inside adapter struct
2365  **/
2366 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work)
2367 {
2368         struct e1000_adapter *adapter = container_of(work,
2369                                                      struct e1000_adapter,
2370                                                      fifo_stall_task.work);
2371         struct e1000_hw *hw = &adapter->hw;
2372         struct net_device *netdev = adapter->netdev;
2373         u32 tctl;
2374
2375         if (atomic_read(&adapter->tx_fifo_stall)) {
2376                 if ((er32(TDT) == er32(TDH)) &&
2377                    (er32(TDFT) == er32(TDFH)) &&
2378                    (er32(TDFTS) == er32(TDFHS))) {
2379                         tctl = er32(TCTL);
2380                         ew32(TCTL, tctl & ~E1000_TCTL_EN);
2381                         ew32(TDFT, adapter->tx_head_addr);
2382                         ew32(TDFH, adapter->tx_head_addr);
2383                         ew32(TDFTS, adapter->tx_head_addr);
2384                         ew32(TDFHS, adapter->tx_head_addr);
2385                         ew32(TCTL, tctl);
2386                         E1000_WRITE_FLUSH();
2387
2388                         adapter->tx_fifo_head = 0;
2389                         atomic_set(&adapter->tx_fifo_stall, 0);
2390                         netif_wake_queue(netdev);
2391                 } else if (!test_bit(__E1000_DOWN, &adapter->flags)) {
2392                         schedule_delayed_work(&adapter->fifo_stall_task, 1);
2393                 }
2394         }
2395 }
2396
2397 bool e1000_has_link(struct e1000_adapter *adapter)
2398 {
2399         struct e1000_hw *hw = &adapter->hw;
2400         bool link_active = false;
2401
2402         /* get_link_status is set on LSC (link status) interrupt or rx
2403          * sequence error interrupt (except on intel ce4100).
2404          * get_link_status will stay false until the
2405          * e1000_check_for_link establishes link for copper adapters
2406          * ONLY
2407          */
2408         switch (hw->media_type) {
2409         case e1000_media_type_copper:
2410                 if (hw->mac_type == e1000_ce4100)
2411                         hw->get_link_status = 1;
2412                 if (hw->get_link_status) {
2413                         e1000_check_for_link(hw);
2414                         link_active = !hw->get_link_status;
2415                 } else {
2416                         link_active = true;
2417                 }
2418                 break;
2419         case e1000_media_type_fiber:
2420                 e1000_check_for_link(hw);
2421                 link_active = !!(er32(STATUS) & E1000_STATUS_LU);
2422                 break;
2423         case e1000_media_type_internal_serdes:
2424                 e1000_check_for_link(hw);
2425                 link_active = hw->serdes_has_link;
2426                 break;
2427         default:
2428                 break;
2429         }
2430
2431         return link_active;
2432 }
2433
2434 /**
2435  * e1000_watchdog - work function
2436  * @work: work struct contained inside adapter struct
2437  **/
2438 static void e1000_watchdog(struct work_struct *work)
2439 {
2440         struct e1000_adapter *adapter = container_of(work,
2441                                                      struct e1000_adapter,
2442                                                      watchdog_task.work);
2443         struct e1000_hw *hw = &adapter->hw;
2444         struct net_device *netdev = adapter->netdev;
2445         struct e1000_tx_ring *txdr = adapter->tx_ring;
2446         u32 link, tctl;
2447
2448         link = e1000_has_link(adapter);
2449         if ((netif_carrier_ok(netdev)) && link)
2450                 goto link_up;
2451
2452         if (link) {
2453                 if (!netif_carrier_ok(netdev)) {
2454                         u32 ctrl;
2455                         bool txb2b = true;
2456                         /* update snapshot of PHY registers on LSC */
2457                         e1000_get_speed_and_duplex(hw,
2458                                                    &adapter->link_speed,
2459                                                    &adapter->link_duplex);
2460
2461                         ctrl = er32(CTRL);
2462                         pr_info("%s NIC Link is Up %d Mbps %s, "
2463                                 "Flow Control: %s\n",
2464                                 netdev->name,
2465                                 adapter->link_speed,
2466                                 adapter->link_duplex == FULL_DUPLEX ?
2467                                 "Full Duplex" : "Half Duplex",
2468                                 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2469                                 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2470                                 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2471                                 E1000_CTRL_TFCE) ? "TX" : "None")));
2472
2473                         /* adjust timeout factor according to speed/duplex */
2474                         adapter->tx_timeout_factor = 1;
2475                         switch (adapter->link_speed) {
2476                         case SPEED_10:
2477                                 txb2b = false;
2478                                 adapter->tx_timeout_factor = 16;
2479                                 break;
2480                         case SPEED_100:
2481                                 txb2b = false;
2482                                 /* maybe add some timeout factor ? */
2483                                 break;
2484                         }
2485
2486                         /* enable transmits in the hardware */
2487                         tctl = er32(TCTL);
2488                         tctl |= E1000_TCTL_EN;
2489                         ew32(TCTL, tctl);
2490
2491                         netif_carrier_on(netdev);
2492                         if (!test_bit(__E1000_DOWN, &adapter->flags))
2493                                 schedule_delayed_work(&adapter->phy_info_task,
2494                                                       2 * HZ);
2495                         adapter->smartspeed = 0;
2496                 }
2497         } else {
2498                 if (netif_carrier_ok(netdev)) {
2499                         adapter->link_speed = 0;
2500                         adapter->link_duplex = 0;
2501                         pr_info("%s NIC Link is Down\n",
2502                                 netdev->name);
2503                         netif_carrier_off(netdev);
2504
2505                         if (!test_bit(__E1000_DOWN, &adapter->flags))
2506                                 schedule_delayed_work(&adapter->phy_info_task,
2507                                                       2 * HZ);
2508                 }
2509
2510                 e1000_smartspeed(adapter);
2511         }
2512
2513 link_up:
2514         e1000_update_stats(adapter);
2515
2516         hw->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
2517         adapter->tpt_old = adapter->stats.tpt;
2518         hw->collision_delta = adapter->stats.colc - adapter->colc_old;
2519         adapter->colc_old = adapter->stats.colc;
2520
2521         adapter->gorcl = adapter->stats.gorcl - adapter->gorcl_old;
2522         adapter->gorcl_old = adapter->stats.gorcl;
2523         adapter->gotcl = adapter->stats.gotcl - adapter->gotcl_old;
2524         adapter->gotcl_old = adapter->stats.gotcl;
2525
2526         e1000_update_adaptive(hw);
2527
2528         if (!netif_carrier_ok(netdev)) {
2529                 if (E1000_DESC_UNUSED(txdr) + 1 < txdr->count) {
2530                         /* We've lost link, so the controller stops DMA,
2531                          * but we've got queued Tx work that's never going
2532                          * to get done, so reset controller to flush Tx.
2533                          * (Do the reset outside of interrupt context).
2534                          */
2535                         adapter->tx_timeout_count++;
2536                         schedule_work(&adapter->reset_task);
2537                         /* exit immediately since reset is imminent */
2538                         return;
2539                 }
2540         }
2541
2542         /* Simple mode for Interrupt Throttle Rate (ITR) */
2543         if (hw->mac_type >= e1000_82540 && adapter->itr_setting == 4) {
2544                 /* Symmetric Tx/Rx gets a reduced ITR=2000;
2545                  * Total asymmetrical Tx or Rx gets ITR=8000;
2546                  * everyone else is between 2000-8000.
2547                  */
2548                 u32 goc = (adapter->gotcl + adapter->gorcl) / 10000;
2549                 u32 dif = (adapter->gotcl > adapter->gorcl ?
2550                             adapter->gotcl - adapter->gorcl :
2551                             adapter->gorcl - adapter->gotcl) / 10000;
2552                 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
2553
2554                 ew32(ITR, 1000000000 / (itr * 256));
2555         }
2556
2557         /* Cause software interrupt to ensure rx ring is cleaned */
2558         ew32(ICS, E1000_ICS_RXDMT0);
2559
2560         /* Force detection of hung controller every watchdog period */
2561         adapter->detect_tx_hung = true;
2562
2563         /* Reschedule the task */
2564         if (!test_bit(__E1000_DOWN, &adapter->flags))
2565                 schedule_delayed_work(&adapter->watchdog_task, 2 * HZ);
2566 }
2567
2568 enum latency_range {
2569         lowest_latency = 0,
2570         low_latency = 1,
2571         bulk_latency = 2,
2572         latency_invalid = 255
2573 };
2574
2575 /**
2576  * e1000_update_itr - update the dynamic ITR value based on statistics
2577  * @adapter: pointer to adapter
2578  * @itr_setting: current adapter->itr
2579  * @packets: the number of packets during this measurement interval
2580  * @bytes: the number of bytes during this measurement interval
2581  *
2582  *      Stores a new ITR value based on packets and byte
2583  *      counts during the last interrupt.  The advantage of per interrupt
2584  *      computation is faster updates and more accurate ITR for the current
2585  *      traffic pattern.  Constants in this function were computed
2586  *      based on theoretical maximum wire speed and thresholds were set based
2587  *      on testing data as well as attempting to minimize response time
2588  *      while increasing bulk throughput.
2589  *      this functionality is controlled by the InterruptThrottleRate module
2590  *      parameter (see e1000_param.c)
2591  **/
2592 static unsigned int e1000_update_itr(struct e1000_adapter *adapter,
2593                                      u16 itr_setting, int packets, int bytes)
2594 {
2595         unsigned int retval = itr_setting;
2596         struct e1000_hw *hw = &adapter->hw;
2597
2598         if (unlikely(hw->mac_type < e1000_82540))
2599                 goto update_itr_done;
2600
2601         if (packets == 0)
2602                 goto update_itr_done;
2603
2604         switch (itr_setting) {
2605         case lowest_latency:
2606                 /* jumbo frames get bulk treatment*/
2607                 if (bytes/packets > 8000)
2608                         retval = bulk_latency;
2609                 else if ((packets < 5) && (bytes > 512))
2610                         retval = low_latency;
2611                 break;
2612         case low_latency:  /* 50 usec aka 20000 ints/s */
2613                 if (bytes > 10000) {
2614                         /* jumbo frames need bulk latency setting */
2615                         if (bytes/packets > 8000)
2616                                 retval = bulk_latency;
2617                         else if ((packets < 10) || ((bytes/packets) > 1200))
2618                                 retval = bulk_latency;
2619                         else if ((packets > 35))
2620                                 retval = lowest_latency;
2621                 } else if (bytes/packets > 2000)
2622                         retval = bulk_latency;
2623                 else if (packets <= 2 && bytes < 512)
2624                         retval = lowest_latency;
2625                 break;
2626         case bulk_latency: /* 250 usec aka 4000 ints/s */
2627                 if (bytes > 25000) {
2628                         if (packets > 35)
2629                                 retval = low_latency;
2630                 } else if (bytes < 6000) {
2631                         retval = low_latency;
2632                 }
2633                 break;
2634         }
2635
2636 update_itr_done:
2637         return retval;
2638 }
2639
2640 static void e1000_set_itr(struct e1000_adapter *adapter)
2641 {
2642         struct e1000_hw *hw = &adapter->hw;
2643         u16 current_itr;
2644         u32 new_itr = adapter->itr;
2645
2646         if (unlikely(hw->mac_type < e1000_82540))
2647                 return;
2648
2649         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2650         if (unlikely(adapter->link_speed != SPEED_1000)) {
2651                 current_itr = 0;
2652                 new_itr = 4000;
2653                 goto set_itr_now;
2654         }
2655
2656         adapter->tx_itr = e1000_update_itr(adapter, adapter->tx_itr,
2657                                            adapter->total_tx_packets,
2658                                            adapter->total_tx_bytes);
2659         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2660         if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2661                 adapter->tx_itr = low_latency;
2662
2663         adapter->rx_itr = e1000_update_itr(adapter, adapter->rx_itr,
2664                                            adapter->total_rx_packets,
2665                                            adapter->total_rx_bytes);
2666         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2667         if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2668                 adapter->rx_itr = low_latency;
2669
2670         current_itr = max(adapter->rx_itr, adapter->tx_itr);
2671
2672         switch (current_itr) {
2673         /* counts and packets in update_itr are dependent on these numbers */
2674         case lowest_latency:
2675                 new_itr = 70000;
2676                 break;
2677         case low_latency:
2678                 new_itr = 20000; /* aka hwitr = ~200 */
2679                 break;
2680         case bulk_latency:
2681                 new_itr = 4000;
2682                 break;
2683         default:
2684                 break;
2685         }
2686
2687 set_itr_now:
2688         if (new_itr != adapter->itr) {
2689                 /* this attempts to bias the interrupt rate towards Bulk
2690                  * by adding intermediate steps when interrupt rate is
2691                  * increasing
2692                  */
2693                 new_itr = new_itr > adapter->itr ?
2694                           min(adapter->itr + (new_itr >> 2), new_itr) :
2695                           new_itr;
2696                 adapter->itr = new_itr;
2697                 ew32(ITR, 1000000000 / (new_itr * 256));
2698         }
2699 }
2700
2701 #define E1000_TX_FLAGS_CSUM             0x00000001
2702 #define E1000_TX_FLAGS_VLAN             0x00000002
2703 #define E1000_TX_FLAGS_TSO              0x00000004
2704 #define E1000_TX_FLAGS_IPV4             0x00000008
2705 #define E1000_TX_FLAGS_NO_FCS           0x00000010
2706 #define E1000_TX_FLAGS_VLAN_MASK        0xffff0000
2707 #define E1000_TX_FLAGS_VLAN_SHIFT       16
2708
2709 static int e1000_tso(struct e1000_adapter *adapter,
2710                      struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2711                      __be16 protocol)
2712 {
2713         struct e1000_context_desc *context_desc;
2714         struct e1000_tx_buffer *buffer_info;
2715         unsigned int i;
2716         u32 cmd_length = 0;
2717         u16 ipcse = 0, tucse, mss;
2718         u8 ipcss, ipcso, tucss, tucso, hdr_len;
2719
2720         if (skb_is_gso(skb)) {
2721                 int err;
2722
2723                 err = skb_cow_head(skb, 0);
2724                 if (err < 0)
2725                         return err;
2726
2727                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2728                 mss = skb_shinfo(skb)->gso_size;
2729                 if (protocol == htons(ETH_P_IP)) {
2730                         struct iphdr *iph = ip_hdr(skb);
2731                         iph->tot_len = 0;
2732                         iph->check = 0;
2733                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2734                                                                  iph->daddr, 0,
2735                                                                  IPPROTO_TCP,
2736                                                                  0);
2737                         cmd_length = E1000_TXD_CMD_IP;
2738                         ipcse = skb_transport_offset(skb) - 1;
2739                 } else if (skb_is_gso_v6(skb)) {
2740                         ipv6_hdr(skb)->payload_len = 0;
2741                         tcp_hdr(skb)->check =
2742                                 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2743                                                  &ipv6_hdr(skb)->daddr,
2744                                                  0, IPPROTO_TCP, 0);
2745                         ipcse = 0;
2746                 }
2747                 ipcss = skb_network_offset(skb);
2748                 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
2749                 tucss = skb_transport_offset(skb);
2750                 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
2751                 tucse = 0;
2752
2753                 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
2754                                E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
2755
2756                 i = tx_ring->next_to_use;
2757                 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2758                 buffer_info = &tx_ring->buffer_info[i];
2759
2760                 context_desc->lower_setup.ip_fields.ipcss  = ipcss;
2761                 context_desc->lower_setup.ip_fields.ipcso  = ipcso;
2762                 context_desc->lower_setup.ip_fields.ipcse  = cpu_to_le16(ipcse);
2763                 context_desc->upper_setup.tcp_fields.tucss = tucss;
2764                 context_desc->upper_setup.tcp_fields.tucso = tucso;
2765                 context_desc->upper_setup.tcp_fields.tucse = cpu_to_le16(tucse);
2766                 context_desc->tcp_seg_setup.fields.mss     = cpu_to_le16(mss);
2767                 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
2768                 context_desc->cmd_and_length = cpu_to_le32(cmd_length);
2769
2770                 buffer_info->time_stamp = jiffies;
2771                 buffer_info->next_to_watch = i;
2772
2773                 if (++i == tx_ring->count)
2774                         i = 0;
2775
2776                 tx_ring->next_to_use = i;
2777
2778                 return true;
2779         }
2780         return false;
2781 }
2782
2783 static bool e1000_tx_csum(struct e1000_adapter *adapter,
2784                           struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2785                           __be16 protocol)
2786 {
2787         struct e1000_context_desc *context_desc;
2788         struct e1000_tx_buffer *buffer_info;
2789         unsigned int i;
2790         u8 css;
2791         u32 cmd_len = E1000_TXD_CMD_DEXT;
2792
2793         if (skb->ip_summed != CHECKSUM_PARTIAL)
2794                 return false;
2795
2796         switch (protocol) {
2797         case cpu_to_be16(ETH_P_IP):
2798                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2799                         cmd_len |= E1000_TXD_CMD_TCP;
2800                 break;
2801         case cpu_to_be16(ETH_P_IPV6):
2802                 /* XXX not handling all IPV6 headers */
2803                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2804                         cmd_len |= E1000_TXD_CMD_TCP;
2805                 break;
2806         default:
2807                 if (unlikely(net_ratelimit()))
2808                         e_warn(drv, "checksum_partial proto=%x!\n",
2809                                skb->protocol);
2810                 break;
2811         }
2812
2813         css = skb_checksum_start_offset(skb);
2814
2815         i = tx_ring->next_to_use;
2816         buffer_info = &tx_ring->buffer_info[i];
2817         context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2818
2819         context_desc->lower_setup.ip_config = 0;
2820         context_desc->upper_setup.tcp_fields.tucss = css;
2821         context_desc->upper_setup.tcp_fields.tucso =
2822                 css + skb->csum_offset;
2823         context_desc->upper_setup.tcp_fields.tucse = 0;
2824         context_desc->tcp_seg_setup.data = 0;
2825         context_desc->cmd_and_length = cpu_to_le32(cmd_len);
2826
2827         buffer_info->time_stamp = jiffies;
2828         buffer_info->next_to_watch = i;
2829
2830         if (unlikely(++i == tx_ring->count))
2831                 i = 0;
2832
2833         tx_ring->next_to_use = i;
2834
2835         return true;
2836 }
2837
2838 #define E1000_MAX_TXD_PWR       12
2839 #define E1000_MAX_DATA_PER_TXD  (1<<E1000_MAX_TXD_PWR)
2840
2841 static int e1000_tx_map(struct e1000_adapter *adapter,
2842                         struct e1000_tx_ring *tx_ring,
2843                         struct sk_buff *skb, unsigned int first,
2844                         unsigned int max_per_txd, unsigned int nr_frags,
2845                         unsigned int mss)
2846 {
2847         struct e1000_hw *hw = &adapter->hw;
2848         struct pci_dev *pdev = adapter->pdev;
2849         struct e1000_tx_buffer *buffer_info;
2850         unsigned int len = skb_headlen(skb);
2851         unsigned int offset = 0, size, count = 0, i;
2852         unsigned int f, bytecount, segs;
2853
2854         i = tx_ring->next_to_use;
2855
2856         while (len) {
2857                 buffer_info = &tx_ring->buffer_info[i];
2858                 size = min(len, max_per_txd);
2859                 /* Workaround for Controller erratum --
2860                  * descriptor for non-tso packet in a linear SKB that follows a
2861                  * tso gets written back prematurely before the data is fully
2862                  * DMA'd to the controller
2863                  */
2864                 if (!skb->data_len && tx_ring->last_tx_tso &&
2865                     !skb_is_gso(skb)) {
2866                         tx_ring->last_tx_tso = false;
2867                         size -= 4;
2868                 }
2869
2870                 /* Workaround for premature desc write-backs
2871                  * in TSO mode.  Append 4-byte sentinel desc
2872                  */
2873                 if (unlikely(mss && !nr_frags && size == len && size > 8))
2874                         size -= 4;
2875                 /* work-around for errata 10 and it applies
2876                  * to all controllers in PCI-X mode
2877                  * The fix is to make sure that the first descriptor of a
2878                  * packet is smaller than 2048 - 16 - 16 (or 2016) bytes
2879                  */
2880                 if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
2881                              (size > 2015) && count == 0))
2882                         size = 2015;
2883
2884                 /* Workaround for potential 82544 hang in PCI-X.  Avoid
2885                  * terminating buffers within evenly-aligned dwords.
2886                  */
2887                 if (unlikely(adapter->pcix_82544 &&
2888                    !((unsigned long)(skb->data + offset + size - 1) & 4) &&
2889                    size > 4))
2890                         size -= 4;
2891
2892                 buffer_info->length = size;
2893                 /* set time_stamp *before* dma to help avoid a possible race */
2894                 buffer_info->time_stamp = jiffies;
2895                 buffer_info->mapped_as_page = false;
2896                 buffer_info->dma = dma_map_single(&pdev->dev,
2897                                                   skb->data + offset,
2898                                                   size, DMA_TO_DEVICE);
2899                 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2900                         goto dma_error;
2901                 buffer_info->next_to_watch = i;
2902
2903                 len -= size;
2904                 offset += size;
2905                 count++;
2906                 if (len) {
2907                         i++;
2908                         if (unlikely(i == tx_ring->count))
2909                                 i = 0;
2910                 }
2911         }
2912
2913         for (f = 0; f < nr_frags; f++) {
2914                 const struct skb_frag_struct *frag;
2915
2916                 frag = &skb_shinfo(skb)->frags[f];
2917                 len = skb_frag_size(frag);
2918                 offset = 0;
2919
2920                 while (len) {
2921                         unsigned long bufend;
2922                         i++;
2923                         if (unlikely(i == tx_ring->count))
2924                                 i = 0;
2925
2926                         buffer_info = &tx_ring->buffer_info[i];
2927                         size = min(len, max_per_txd);
2928                         /* Workaround for premature desc write-backs
2929                          * in TSO mode.  Append 4-byte sentinel desc
2930                          */
2931                         if (unlikely(mss && f == (nr_frags-1) &&
2932                             size == len && size > 8))
2933                                 size -= 4;
2934                         /* Workaround for potential 82544 hang in PCI-X.
2935                          * Avoid terminating buffers within evenly-aligned
2936                          * dwords.
2937                          */
2938                         bufend = (unsigned long)
2939                                 page_to_phys(skb_frag_page(frag));
2940                         bufend += offset + size - 1;
2941                         if (unlikely(adapter->pcix_82544 &&
2942                                      !(bufend & 4) &&
2943                                      size > 4))
2944                                 size -= 4;
2945
2946                         buffer_info->length = size;
2947                         buffer_info->time_stamp = jiffies;
2948                         buffer_info->mapped_as_page = true;
2949                         buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
2950                                                 offset, size, DMA_TO_DEVICE);
2951                         if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2952                                 goto dma_error;
2953                         buffer_info->next_to_watch = i;
2954
2955                         len -= size;
2956                         offset += size;
2957                         count++;
2958                 }
2959         }
2960
2961         segs = skb_shinfo(skb)->gso_segs ?: 1;
2962         /* multiply data chunks by size of headers */
2963         bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
2964
2965         tx_ring->buffer_info[i].skb = skb;
2966         tx_ring->buffer_info[i].segs = segs;
2967         tx_ring->buffer_info[i].bytecount = bytecount;
2968         tx_ring->buffer_info[first].next_to_watch = i;
2969
2970         return count;
2971
2972 dma_error:
2973         dev_err(&pdev->dev, "TX DMA map failed\n");
2974         buffer_info->dma = 0;
2975         if (count)
2976                 count--;
2977
2978         while (count--) {
2979                 if (i == 0)
2980                         i += tx_ring->count;
2981                 i--;
2982                 buffer_info = &tx_ring->buffer_info[i];
2983                 e1000_unmap_and_free_tx_resource(adapter, buffer_info);
2984         }
2985
2986         return 0;
2987 }
2988
2989 static void e1000_tx_queue(struct e1000_adapter *adapter,
2990                            struct e1000_tx_ring *tx_ring, int tx_flags,
2991                            int count)
2992 {
2993         struct e1000_tx_desc *tx_desc = NULL;
2994         struct e1000_tx_buffer *buffer_info;
2995         u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
2996         unsigned int i;
2997
2998         if (likely(tx_flags & E1000_TX_FLAGS_TSO)) {
2999                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
3000                              E1000_TXD_CMD_TSE;
3001                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3002
3003                 if (likely(tx_flags & E1000_TX_FLAGS_IPV4))
3004                         txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3005         }
3006
3007         if (likely(tx_flags & E1000_TX_FLAGS_CSUM)) {
3008                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3009                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3010         }
3011
3012         if (unlikely(tx_flags & E1000_TX_FLAGS_VLAN)) {
3013                 txd_lower |= E1000_TXD_CMD_VLE;
3014                 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
3015         }
3016
3017         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
3018                 txd_lower &= ~(E1000_TXD_CMD_IFCS);
3019
3020         i = tx_ring->next_to_use;
3021
3022         while (count--) {
3023                 buffer_info = &tx_ring->buffer_info[i];
3024                 tx_desc = E1000_TX_DESC(*tx_ring, i);
3025                 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
3026                 tx_desc->lower.data =
3027                         cpu_to_le32(txd_lower | buffer_info->length);
3028                 tx_desc->upper.data = cpu_to_le32(txd_upper);
3029                 if (unlikely(++i == tx_ring->count))
3030                         i = 0;
3031         }
3032
3033         tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
3034
3035         /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */
3036         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
3037                 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));
3038
3039         /* Force memory writes to complete before letting h/w
3040          * know there are new descriptors to fetch.  (Only
3041          * applicable for weak-ordered memory model archs,
3042          * such as IA-64).
3043          */
3044         wmb();
3045
3046         tx_ring->next_to_use = i;
3047 }
3048
3049 /* 82547 workaround to avoid controller hang in half-duplex environment.
3050  * The workaround is to avoid queuing a large packet that would span
3051  * the internal Tx FIFO ring boundary by notifying the stack to resend
3052  * the packet at a later time.  This gives the Tx FIFO an opportunity to
3053  * flush all packets.  When that occurs, we reset the Tx FIFO pointers
3054  * to the beginning of the Tx FIFO.
3055  */
3056
3057 #define E1000_FIFO_HDR                  0x10
3058 #define E1000_82547_PAD_LEN             0x3E0
3059
3060 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
3061                                        struct sk_buff *skb)
3062 {
3063         u32 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
3064         u32 skb_fifo_len = skb->len + E1000_FIFO_HDR;
3065
3066         skb_fifo_len = ALIGN(skb_fifo_len, E1000_FIFO_HDR);
3067
3068         if (adapter->link_duplex != HALF_DUPLEX)
3069                 goto no_fifo_stall_required;
3070
3071         if (atomic_read(&adapter->tx_fifo_stall))
3072                 return 1;
3073
3074         if (skb_fifo_len >= (E1000_82547_PAD_LEN + fifo_space)) {
3075                 atomic_set(&adapter->tx_fifo_stall, 1);
3076                 return 1;
3077         }
3078
3079 no_fifo_stall_required:
3080         adapter->tx_fifo_head += skb_fifo_len;
3081         if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
3082                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
3083         return 0;
3084 }
3085
3086 static int __e1000_maybe_stop_tx(struct net_device *netdev, int size)
3087 {
3088         struct e1000_adapter *adapter = netdev_priv(netdev);
3089         struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3090
3091         netif_stop_queue(netdev);
3092         /* Herbert's original patch had:
3093          *  smp_mb__after_netif_stop_queue();
3094          * but since that doesn't exist yet, just open code it.
3095          */
3096         smp_mb();
3097
3098         /* We need to check again in a case another CPU has just
3099          * made room available.
3100          */
3101         if (likely(E1000_DESC_UNUSED(tx_ring) < size))
3102                 return -EBUSY;
3103
3104         /* A reprieve! */
3105         netif_start_queue(netdev);
3106         ++adapter->restart_queue;
3107         return 0;
3108 }
3109
3110 static int e1000_maybe_stop_tx(struct net_device *netdev,
3111                                struct e1000_tx_ring *tx_ring, int size)
3112 {
3113         if (likely(E1000_DESC_UNUSED(tx_ring) >= size))
3114                 return 0;
3115         return __e1000_maybe_stop_tx(netdev, size);
3116 }
3117
3118 #define TXD_USE_COUNT(S, X) (((S) + ((1 << (X)) - 1)) >> (X))
3119 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
3120                                     struct net_device *netdev)
3121 {
3122         struct e1000_adapter *adapter = netdev_priv(netdev);
3123         struct e1000_hw *hw = &adapter->hw;
3124         struct e1000_tx_ring *tx_ring;
3125         unsigned int first, max_per_txd = E1000_MAX_DATA_PER_TXD;
3126         unsigned int max_txd_pwr = E1000_MAX_TXD_PWR;
3127         unsigned int tx_flags = 0;
3128         unsigned int len = skb_headlen(skb);
3129         unsigned int nr_frags;
3130         unsigned int mss;
3131         int count = 0;
3132         int tso;
3133         unsigned int f;
3134         __be16 protocol = vlan_get_protocol(skb);
3135
3136         /* This goes back to the question of how to logically map a Tx queue
3137          * to a flow.  Right now, performance is impacted slightly negatively
3138          * if using multiple Tx queues.  If the stack breaks away from a
3139          * single qdisc implementation, we can look at this again.
3140          */
3141         tx_ring = adapter->tx_ring;
3142
3143         /* On PCI/PCI-X HW, if packet size is less than ETH_ZLEN,
3144          * packets may get corrupted during padding by HW.
3145          * To WA this issue, pad all small packets manually.
3146          */
3147         if (eth_skb_pad(skb))
3148                 return NETDEV_TX_OK;
3149
3150         mss = skb_shinfo(skb)->gso_size;
3151         /* The controller does a simple calculation to
3152          * make sure there is enough room in the FIFO before
3153          * initiating the DMA for each buffer.  The calc is:
3154          * 4 = ceil(buffer len/mss).  To make sure we don't
3155          * overrun the FIFO, adjust the max buffer len if mss
3156          * drops.
3157          */
3158         if (mss) {
3159                 u8 hdr_len;
3160                 max_per_txd = min(mss << 2, max_per_txd);
3161                 max_txd_pwr = fls(max_per_txd) - 1;
3162
3163                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
3164                 if (skb->data_len && hdr_len == len) {
3165                         switch (hw->mac_type) {
3166                                 unsigned int pull_size;
3167                         case e1000_82544:
3168                                 /* Make sure we have room to chop off 4 bytes,
3169                                  * and that the end alignment will work out to
3170                                  * this hardware's requirements
3171                                  * NOTE: this is a TSO only workaround
3172                                  * if end byte alignment not correct move us
3173                                  * into the next dword
3174                                  */
3175                                 if ((unsigned long)(skb_tail_pointer(skb) - 1)
3176                                     & 4)
3177                                         break;
3178                                 /* fall through */
3179                                 pull_size = min((unsigned int)4, skb->data_len);
3180                                 if (!__pskb_pull_tail(skb, pull_size)) {
3181                                         e_err(drv, "__pskb_pull_tail "
3182                                               "failed.\n");
3183                                         dev_kfree_skb_any(skb);
3184                                         return NETDEV_TX_OK;
3185                                 }
3186                                 len = skb_headlen(skb);
3187                                 break;
3188                         default:
3189                                 /* do nothing */
3190                                 break;
3191                         }
3192                 }
3193         }
3194
3195         /* reserve a descriptor for the offload context */
3196         if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
3197                 count++;
3198         count++;
3199
3200         /* Controller Erratum workaround */
3201         if (!skb->data_len && tx_ring->last_tx_tso && !skb_is_gso(skb))
3202                 count++;
3203
3204         count += TXD_USE_COUNT(len, max_txd_pwr);
3205
3206         if (adapter->pcix_82544)
3207                 count++;
3208
3209         /* work-around for errata 10 and it applies to all controllers
3210          * in PCI-X mode, so add one more descriptor to the count
3211          */
3212         if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
3213                         (len > 2015)))
3214                 count++;
3215
3216         nr_frags = skb_shinfo(skb)->nr_frags;
3217         for (f = 0; f < nr_frags; f++)
3218                 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->frags[f]),
3219                                        max_txd_pwr);
3220         if (adapter->pcix_82544)
3221                 count += nr_frags;
3222
3223         /* need: count + 2 desc gap to keep tail from touching
3224          * head, otherwise try next time
3225          */
3226         if (unlikely(e1000_maybe_stop_tx(netdev, tx_ring, count + 2)))
3227                 return NETDEV_TX_BUSY;
3228
3229         if (unlikely((hw->mac_type == e1000_82547) &&
3230                      (e1000_82547_fifo_workaround(adapter, skb)))) {
3231                 netif_stop_queue(netdev);
3232                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3233                         schedule_delayed_work(&adapter->fifo_stall_task, 1);
3234                 return NETDEV_TX_BUSY;
3235         }
3236
3237         if (skb_vlan_tag_present(skb)) {
3238                 tx_flags |= E1000_TX_FLAGS_VLAN;
3239                 tx_flags |= (skb_vlan_tag_get(skb) <<
3240                              E1000_TX_FLAGS_VLAN_SHIFT);
3241         }
3242
3243         first = tx_ring->next_to_use;
3244
3245         tso = e1000_tso(adapter, tx_ring, skb, protocol);
3246         if (tso < 0) {
3247                 dev_kfree_skb_any(skb);
3248                 return NETDEV_TX_OK;
3249         }
3250
3251         if (likely(tso)) {
3252                 if (likely(hw->mac_type != e1000_82544))
3253                         tx_ring->last_tx_tso = true;
3254                 tx_flags |= E1000_TX_FLAGS_TSO;
3255         } else if (likely(e1000_tx_csum(adapter, tx_ring, skb, protocol)))
3256                 tx_flags |= E1000_TX_FLAGS_CSUM;
3257
3258         if (protocol == htons(ETH_P_IP))
3259                 tx_flags |= E1000_TX_FLAGS_IPV4;
3260
3261         if (unlikely(skb->no_fcs))
3262                 tx_flags |= E1000_TX_FLAGS_NO_FCS;
3263
3264         count = e1000_tx_map(adapter, tx_ring, skb, first, max_per_txd,
3265                              nr_frags, mss);
3266
3267         if (count) {
3268                 /* The descriptors needed is higher than other Intel drivers
3269                  * due to a number of workarounds.  The breakdown is below:
3270                  * Data descriptors: MAX_SKB_FRAGS + 1
3271                  * Context Descriptor: 1
3272                  * Keep head from touching tail: 2
3273                  * Workarounds: 3
3274                  */
3275                 int desc_needed = MAX_SKB_FRAGS + 7;
3276
3277                 netdev_sent_queue(netdev, skb->len);
3278                 skb_tx_timestamp(skb);
3279
3280                 e1000_tx_queue(adapter, tx_ring, tx_flags, count);
3281
3282                 /* 82544 potentially requires twice as many data descriptors
3283                  * in order to guarantee buffers don't end on evenly-aligned
3284                  * dwords
3285                  */
3286                 if (adapter->pcix_82544)
3287                         desc_needed += MAX_SKB_FRAGS + 1;
3288
3289                 /* Make sure there is space in the ring for the next send. */
3290                 e1000_maybe_stop_tx(netdev, tx_ring, desc_needed);
3291
3292                 if (!skb->xmit_more ||
3293                     netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) {
3294                         writel(tx_ring->next_to_use, hw->hw_addr + tx_ring->tdt);
3295                         /* we need this if more than one processor can write to
3296                          * our tail at a time, it synchronizes IO on IA64/Altix
3297                          * systems
3298                          */
3299                         mmiowb();
3300                 }
3301         } else {
3302                 dev_kfree_skb_any(skb);
3303                 tx_ring->buffer_info[first].time_stamp = 0;
3304                 tx_ring->next_to_use = first;
3305         }
3306
3307         return NETDEV_TX_OK;
3308 }
3309
3310 #define NUM_REGS 38 /* 1 based count */
3311 static void e1000_regdump(struct e1000_adapter *adapter)
3312 {
3313         struct e1000_hw *hw = &adapter->hw;
3314         u32 regs[NUM_REGS];
3315         u32 *regs_buff = regs;
3316         int i = 0;
3317
3318         static const char * const reg_name[] = {
3319                 "CTRL",  "STATUS",
3320                 "RCTL", "RDLEN", "RDH", "RDT", "RDTR",
3321                 "TCTL", "TDBAL", "TDBAH", "TDLEN", "TDH", "TDT",
3322                 "TIDV", "TXDCTL", "TADV", "TARC0",
3323                 "TDBAL1", "TDBAH1", "TDLEN1", "TDH1", "TDT1",
3324                 "TXDCTL1", "TARC1",
3325                 "CTRL_EXT", "ERT", "RDBAL", "RDBAH",
3326                 "TDFH", "TDFT", "TDFHS", "TDFTS", "TDFPC",
3327                 "RDFH", "RDFT", "RDFHS", "RDFTS", "RDFPC"
3328         };
3329
3330         regs_buff[0]  = er32(CTRL);
3331         regs_buff[1]  = er32(STATUS);
3332
3333         regs_buff[2]  = er32(RCTL);
3334         regs_buff[3]  = er32(RDLEN);
3335         regs_buff[4]  = er32(RDH);
3336         regs_buff[5]  = er32(RDT);
3337         regs_buff[6]  = er32(RDTR);
3338
3339         regs_buff[7]  = er32(TCTL);
3340         regs_buff[8]  = er32(TDBAL);
3341         regs_buff[9]  = er32(TDBAH);
3342         regs_buff[10] = er32(TDLEN);
3343         regs_buff[11] = er32(TDH);
3344         regs_buff[12] = er32(TDT);
3345         regs_buff[13] = er32(TIDV);
3346         regs_buff[14] = er32(TXDCTL);
3347         regs_buff[15] = er32(TADV);
3348         regs_buff[16] = er32(TARC0);
3349
3350         regs_buff[17] = er32(TDBAL1);
3351         regs_buff[18] = er32(TDBAH1);
3352         regs_buff[19] = er32(TDLEN1);
3353         regs_buff[20] = er32(TDH1);
3354         regs_buff[21] = er32(TDT1);
3355         regs_buff[22] = er32(TXDCTL1);
3356         regs_buff[23] = er32(TARC1);
3357         regs_buff[24] = er32(CTRL_EXT);
3358         regs_buff[25] = er32(ERT);
3359         regs_buff[26] = er32(RDBAL0);
3360         regs_buff[27] = er32(RDBAH0);
3361         regs_buff[28] = er32(TDFH);
3362         regs_buff[29] = er32(TDFT);
3363         regs_buff[30] = er32(TDFHS);
3364         regs_buff[31] = er32(TDFTS);
3365         regs_buff[32] = er32(TDFPC);
3366         regs_buff[33] = er32(RDFH);
3367         regs_buff[34] = er32(RDFT);
3368         regs_buff[35] = er32(RDFHS);
3369         regs_buff[36] = er32(RDFTS);
3370         regs_buff[37] = er32(RDFPC);
3371
3372         pr_info("Register dump\n");
3373         for (i = 0; i < NUM_REGS; i++)
3374                 pr_info("%-15s  %08x\n", reg_name[i], regs_buff[i]);
3375 }
3376
3377 /*
3378  * e1000_dump: Print registers, tx ring and rx ring
3379  */
3380 static void e1000_dump(struct e1000_adapter *adapter)
3381 {
3382         /* this code doesn't handle multiple rings */
3383         struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3384         struct e1000_rx_ring *rx_ring = adapter->rx_ring;
3385         int i;
3386
3387         if (!netif_msg_hw(adapter))
3388                 return;
3389
3390         /* Print Registers */
3391         e1000_regdump(adapter);
3392
3393         /* transmit dump */
3394         pr_info("TX Desc ring0 dump\n");
3395
3396         /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
3397          *
3398          * Legacy Transmit Descriptor
3399          *   +--------------------------------------------------------------+
3400          * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
3401          *   +--------------------------------------------------------------+
3402          * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
3403          *   +--------------------------------------------------------------+
3404          *   63       48 47        36 35    32 31     24 23    16 15        0
3405          *
3406          * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
3407          *   63      48 47    40 39       32 31             16 15    8 7      0
3408          *   +----------------------------------------------------------------+
3409          * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
3410          *   +----------------------------------------------------------------+
3411          * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
3412          *   +----------------------------------------------------------------+
3413          *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
3414          *
3415          * Extended Data Descriptor (DTYP=0x1)
3416          *   +----------------------------------------------------------------+
3417          * 0 |                     Buffer Address [63:0]                      |
3418          *   +----------------------------------------------------------------+
3419          * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
3420          *   +----------------------------------------------------------------+
3421          *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
3422          */
3423         pr_info("Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3424         pr_info("Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3425
3426         if (!netif_msg_tx_done(adapter))
3427                 goto rx_ring_summary;
3428
3429         for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
3430                 struct e1000_tx_desc *tx_desc = E1000_TX_DESC(*tx_ring, i);
3431                 struct e1000_tx_buffer *buffer_info = &tx_ring->buffer_info[i];
3432                 struct my_u { __le64 a; __le64 b; };
3433                 struct my_u *u = (struct my_u *)tx_desc;
3434                 const char *type;
3435
3436                 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
3437                         type = "NTC/U";
3438                 else if (i == tx_ring->next_to_use)
3439                         type = "NTU";
3440                 else if (i == tx_ring->next_to_clean)
3441                         type = "NTC";
3442                 else
3443                         type = "";
3444
3445                 pr_info("T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X %016llX %p %s\n",
3446                         ((le64_to_cpu(u->b) & (1<<20)) ? 'd' : 'c'), i,
3447                         le64_to_cpu(u->a), le64_to_cpu(u->b),
3448                         (u64)buffer_info->dma, buffer_info->length,
3449                         buffer_info->next_to_watch,
3450                         (u64)buffer_info->time_stamp, buffer_info->skb, type);
3451         }
3452
3453 rx_ring_summary:
3454         /* receive dump */
3455         pr_info("\nRX Desc ring dump\n");
3456
3457         /* Legacy Receive Descriptor Format
3458          *
3459          * +-----------------------------------------------------+
3460          * |                Buffer Address [63:0]                |
3461          * +-----------------------------------------------------+
3462          * | VLAN Tag | Errors | Status 0 | Packet csum | Length |
3463          * +-----------------------------------------------------+
3464          * 63       48 47    40 39      32 31         16 15      0
3465          */
3466         pr_info("R[desc]      [address 63:0  ] [vl er S cks ln] [bi->dma       ] [bi->skb]\n");
3467
3468         if (!netif_msg_rx_status(adapter))
3469                 goto exit;
3470
3471         for (i = 0; rx_ring->desc && (i < rx_ring->count); i++) {
3472                 struct e1000_rx_desc *rx_desc = E1000_RX_DESC(*rx_ring, i);
3473                 struct e1000_rx_buffer *buffer_info = &rx_ring->buffer_info[i];
3474                 struct my_u { __le64 a; __le64 b; };
3475                 struct my_u *u = (struct my_u *)rx_desc;
3476                 const char *type;
3477
3478                 if (i == rx_ring->next_to_use)
3479                         type = "NTU";
3480                 else if (i == rx_ring->next_to_clean)
3481                         type = "NTC";
3482                 else
3483                         type = "";
3484
3485                 pr_info("R[0x%03X]     %016llX %016llX %016llX %p %s\n",
3486                         i, le64_to_cpu(u->a), le64_to_cpu(u->b),
3487                         (u64)buffer_info->dma, buffer_info->rxbuf.data, type);
3488         } /* for */
3489
3490         /* dump the descriptor caches */
3491         /* rx */
3492         pr_info("Rx descriptor cache in 64bit format\n");
3493         for (i = 0x6000; i <= 0x63FF ; i += 0x10) {
3494                 pr_info("R%04X: %08X|%08X %08X|%08X\n",
3495                         i,
3496                         readl(adapter->hw.hw_addr + i+4),
3497                         readl(adapter->hw.hw_addr + i),
3498                         readl(adapter->hw.hw_addr + i+12),
3499                         readl(adapter->hw.hw_addr + i+8));
3500         }
3501         /* tx */
3502         pr_info("Tx descriptor cache in 64bit format\n");
3503         for (i = 0x7000; i <= 0x73FF ; i += 0x10) {
3504                 pr_info("T%04X: %08X|%08X %08X|%08X\n",
3505                         i,
3506                         readl(adapter->hw.hw_addr + i+4),
3507                         readl(adapter->hw.hw_addr + i),
3508                         readl(adapter->hw.hw_addr + i+12),
3509                         readl(adapter->hw.hw_addr + i+8));
3510         }
3511 exit:
3512         return;
3513 }
3514
3515 /**
3516  * e1000_tx_timeout - Respond to a Tx Hang
3517  * @netdev: network interface device structure
3518  **/
3519 static void e1000_tx_timeout(struct net_device *netdev)
3520 {
3521         struct e1000_adapter *adapter = netdev_priv(netdev);
3522
3523         /* Do the reset outside of interrupt context */
3524         adapter->tx_timeout_count++;
3525         schedule_work(&adapter->reset_task);
3526 }
3527
3528 static void e1000_reset_task(struct work_struct *work)
3529 {
3530         struct e1000_adapter *adapter =
3531                 container_of(work, struct e1000_adapter, reset_task);
3532
3533         e_err(drv, "Reset adapter\n");
3534         e1000_reinit_locked(adapter);
3535 }
3536
3537 /**
3538  * e1000_change_mtu - Change the Maximum Transfer Unit
3539  * @netdev: network interface device structure
3540  * @new_mtu: new value for maximum frame size
3541  *
3542  * Returns 0 on success, negative on failure
3543  **/
3544 static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
3545 {
3546         struct e1000_adapter *adapter = netdev_priv(netdev);
3547         struct e1000_hw *hw = &adapter->hw;
3548         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3549
3550         /* Adapter-specific max frame size limits. */
3551         switch (hw->mac_type) {
3552         case e1000_undefined ... e1000_82542_rev2_1:
3553                 if (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)) {
3554                         e_err(probe, "Jumbo Frames not supported.\n");
3555                         return -EINVAL;
3556                 }
3557                 break;
3558         default:
3559                 /* Capable of supporting up to MAX_JUMBO_FRAME_SIZE limit. */
3560                 break;
3561         }
3562
3563         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
3564                 msleep(1);
3565         /* e1000_down has a dependency on max_frame_size */
3566         hw->max_frame_size = max_frame;
3567         if (netif_running(netdev)) {
3568                 /* prevent buffers from being reallocated */
3569                 adapter->alloc_rx_buf = e1000_alloc_dummy_rx_buffers;
3570                 e1000_down(adapter);
3571         }
3572
3573         /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3574          * means we reserve 2 more, this pushes us to allocate from the next
3575          * larger slab size.
3576          * i.e. RXBUFFER_2048 --> size-4096 slab
3577          * however with the new *_jumbo_rx* routines, jumbo receives will use
3578          * fragmented skbs
3579          */
3580
3581         if (max_frame <= E1000_RXBUFFER_2048)
3582                 adapter->rx_buffer_len = E1000_RXBUFFER_2048;
3583         else
3584 #if (PAGE_SIZE >= E1000_RXBUFFER_16384)
3585                 adapter->rx_buffer_len = E1000_RXBUFFER_16384;
3586 #elif (PAGE_SIZE >= E1000_RXBUFFER_4096)
3587                 adapter->rx_buffer_len = PAGE_SIZE;
3588 #endif
3589
3590         /* adjust allocation if LPE protects us, and we aren't using SBP */
3591         if (!hw->tbi_compatibility_on &&
3592             ((max_frame == (ETH_FRAME_LEN + ETH_FCS_LEN)) ||
3593              (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE)))
3594                 adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3595
3596         pr_info("%s changing MTU from %d to %d\n",
3597                 netdev->name, netdev->mtu, new_mtu);
3598         netdev->mtu = new_mtu;
3599
3600         if (netif_running(netdev))
3601                 e1000_up(adapter);
3602         else
3603                 e1000_reset(adapter);
3604
3605         clear_bit(__E1000_RESETTING, &adapter->flags);
3606
3607         return 0;
3608 }
3609
3610 /**
3611  * e1000_update_stats - Update the board statistics counters
3612  * @adapter: board private structure
3613  **/
3614 void e1000_update_stats(struct e1000_adapter *adapter)
3615 {
3616         struct net_device *netdev = adapter->netdev;
3617         struct e1000_hw *hw = &adapter->hw;
3618         struct pci_dev *pdev = adapter->pdev;
3619         unsigned long flags;
3620         u16 phy_tmp;
3621
3622 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3623
3624         /* Prevent stats update while adapter is being reset, or if the pci
3625          * connection is down.
3626          */
3627         if (adapter->link_speed == 0)
3628                 return;
3629         if (pci_channel_offline(pdev))
3630                 return;
3631
3632         spin_lock_irqsave(&adapter->stats_lock, flags);
3633
3634         /* these counters are modified from e1000_tbi_adjust_stats,
3635          * called from the interrupt context, so they must only
3636          * be written while holding adapter->stats_lock
3637          */
3638
3639         adapter->stats.crcerrs += er32(CRCERRS);
3640         adapter->stats.gprc += er32(GPRC);
3641         adapter->stats.gorcl += er32(GORCL);
3642         adapter->stats.gorch += er32(GORCH);
3643         adapter->stats.bprc += er32(BPRC);
3644         adapter->stats.mprc += er32(MPRC);
3645         adapter->stats.roc += er32(ROC);
3646
3647         adapter->stats.prc64 += er32(PRC64);
3648         adapter->stats.prc127 += er32(PRC127);
3649         adapter->stats.prc255 += er32(PRC255);
3650         adapter->stats.prc511 += er32(PRC511);
3651         adapter->stats.prc1023 += er32(PRC1023);
3652         adapter->stats.prc1522 += er32(PRC1522);
3653
3654         adapter->stats.symerrs += er32(SYMERRS);
3655         adapter->stats.mpc += er32(MPC);
3656         adapter->stats.scc += er32(SCC);
3657         adapter->stats.ecol += er32(ECOL);
3658         adapter->stats.mcc += er32(MCC);
3659         adapter->stats.latecol += er32(LATECOL);
3660         adapter->stats.dc += er32(DC);
3661         adapter->stats.sec += er32(SEC);
3662         adapter->stats.rlec += er32(RLEC);
3663         adapter->stats.xonrxc += er32(XONRXC);
3664         adapter->stats.xontxc += er32(XONTXC);
3665         adapter->stats.xoffrxc += er32(XOFFRXC);
3666         adapter->stats.xofftxc += er32(XOFFTXC);
3667         adapter->stats.fcruc += er32(FCRUC);
3668         adapter->stats.gptc += er32(GPTC);
3669         adapter->stats.gotcl += er32(GOTCL);
3670         adapter->stats.gotch += er32(GOTCH);
3671         adapter->stats.rnbc += er32(RNBC);
3672         adapter->stats.ruc += er32(RUC);
3673         adapter->stats.rfc += er32(RFC);
3674         adapter->stats.rjc += er32(RJC);
3675         adapter->stats.torl += er32(TORL);
3676         adapter->stats.torh += er32(TORH);
3677         adapter->stats.totl += er32(TOTL);
3678         adapter->stats.toth += er32(TOTH);
3679         adapter->stats.tpr += er32(TPR);
3680
3681         adapter->stats.ptc64 += er32(PTC64);
3682         adapter->stats.ptc127 += er32(PTC127);
3683         adapter->stats.ptc255 += er32(PTC255);
3684         adapter->stats.ptc511 += er32(PTC511);
3685         adapter->stats.ptc1023 += er32(PTC1023);
3686         adapter->stats.ptc1522 += er32(PTC1522);
3687
3688         adapter->stats.mptc += er32(MPTC);
3689         adapter->stats.bptc += er32(BPTC);
3690
3691         /* used for adaptive IFS */
3692
3693         hw->tx_packet_delta = er32(TPT);
3694         adapter->stats.tpt += hw->tx_packet_delta;
3695         hw->collision_delta = er32(COLC);
3696         adapter->stats.colc += hw->collision_delta;
3697
3698         if (hw->mac_type >= e1000_82543) {
3699                 adapter->stats.algnerrc += er32(ALGNERRC);
3700                 adapter->stats.rxerrc += er32(RXERRC);
3701                 adapter->stats.tncrs += er32(TNCRS);
3702                 adapter->stats.cexterr += er32(CEXTERR);
3703                 adapter->stats.tsctc += er32(TSCTC);
3704                 adapter->stats.tsctfc += er32(TSCTFC);
3705         }
3706
3707         /* Fill out the OS statistics structure */
3708         netdev->stats.multicast = adapter->stats.mprc;
3709         netdev->stats.collisions = adapter->stats.colc;
3710
3711         /* Rx Errors */
3712
3713         /* RLEC on some newer hardware can be incorrect so build
3714          * our own version based on RUC and ROC
3715          */
3716         netdev->stats.rx_errors = adapter->stats.rxerrc +
3717                 adapter->stats.crcerrs + adapter->stats.algnerrc +
3718                 adapter->stats.ruc + adapter->stats.roc +
3719                 adapter->stats.cexterr;
3720         adapter->stats.rlerrc = adapter->stats.ruc + adapter->stats.roc;
3721         netdev->stats.rx_length_errors = adapter->stats.rlerrc;
3722         netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3723         netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3724         netdev->stats.rx_missed_errors = adapter->stats.mpc;
3725
3726         /* Tx Errors */
3727         adapter->stats.txerrc = adapter->stats.ecol + adapter->stats.latecol;
3728         netdev->stats.tx_errors = adapter->stats.txerrc;
3729         netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3730         netdev->stats.tx_window_errors = adapter->stats.latecol;
3731         netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
3732         if (hw->bad_tx_carr_stats_fd &&
3733             adapter->link_duplex == FULL_DUPLEX) {
3734                 netdev->stats.tx_carrier_errors = 0;
3735                 adapter->stats.tncrs = 0;
3736         }
3737
3738         /* Tx Dropped needs to be maintained elsewhere */
3739
3740         /* Phy Stats */
3741         if (hw->media_type == e1000_media_type_copper) {
3742                 if ((adapter->link_speed == SPEED_1000) &&
3743                    (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
3744                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3745                         adapter->phy_stats.idle_errors += phy_tmp;
3746                 }
3747
3748                 if ((hw->mac_type <= e1000_82546) &&
3749                    (hw->phy_type == e1000_phy_m88) &&
3750                    !e1000_read_phy_reg(hw, M88E1000_RX_ERR_CNTR, &phy_tmp))
3751                         adapter->phy_stats.receive_errors += phy_tmp;
3752         }
3753
3754         /* Management Stats */
3755         if (hw->has_smbus) {
3756                 adapter->stats.mgptc += er32(MGTPTC);
3757                 adapter->stats.mgprc += er32(MGTPRC);
3758                 adapter->stats.mgpdc += er32(MGTPDC);
3759         }
3760
3761         spin_unlock_irqrestore(&adapter->stats_lock, flags);
3762 }
3763
3764 /**
3765  * e1000_intr - Interrupt Handler
3766  * @irq: interrupt number
3767  * @data: pointer to a network interface device structure
3768  **/
3769 static irqreturn_t e1000_intr(int irq, void *data)
3770 {
3771         struct net_device *netdev = data;
3772         struct e1000_adapter *adapter = netdev_priv(netdev);
3773         struct e1000_hw *hw = &adapter->hw;
3774         u32 icr = er32(ICR);
3775
3776         if (unlikely((!icr)))
3777                 return IRQ_NONE;  /* Not our interrupt */
3778
3779         /* we might have caused the interrupt, but the above
3780          * read cleared it, and just in case the driver is
3781          * down there is nothing to do so return handled
3782          */
3783         if (unlikely(test_bit(__E1000_DOWN, &adapter->flags)))
3784                 return IRQ_HANDLED;
3785
3786         if (unlikely(icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))) {
3787                 hw->get_link_status = 1;
3788                 /* guard against interrupt when we're going down */
3789                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3790                         schedule_delayed_work(&adapter->watchdog_task, 1);
3791         }
3792
3793         /* disable interrupts, without the synchronize_irq bit */
3794         ew32(IMC, ~0);
3795         E1000_WRITE_FLUSH();
3796
3797         if (likely(napi_schedule_prep(&adapter->napi))) {
3798                 adapter->total_tx_bytes = 0;
3799                 adapter->total_tx_packets = 0;
3800                 adapter->total_rx_bytes = 0;
3801                 adapter->total_rx_packets = 0;
3802                 __napi_schedule(&adapter->napi);
3803         } else {
3804                 /* this really should not happen! if it does it is basically a
3805                  * bug, but not a hard error, so enable ints and continue
3806                  */
3807                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3808                         e1000_irq_enable(adapter);
3809         }
3810
3811         return IRQ_HANDLED;
3812 }
3813
3814 /**
3815  * e1000_clean - NAPI Rx polling callback
3816  * @adapter: board private structure
3817  **/
3818 static int e1000_clean(struct napi_struct *napi, int budget)
3819 {
3820         struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
3821                                                      napi);
3822         int tx_clean_complete = 0, work_done = 0;
3823
3824         tx_clean_complete = e1000_clean_tx_irq(adapter, &adapter->tx_ring[0]);
3825
3826         adapter->clean_rx(adapter, &adapter->rx_ring[0], &work_done, budget);
3827
3828         if (!tx_clean_complete)
3829                 work_done = budget;
3830
3831         /* If budget not fully consumed, exit the polling mode */
3832         if (work_done < budget) {
3833                 if (likely(adapter->itr_setting & 3))
3834                         e1000_set_itr(adapter);
3835                 napi_complete_done(napi, work_done);
3836                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3837                         e1000_irq_enable(adapter);
3838         }
3839
3840         return work_done;
3841 }
3842
3843 /**
3844  * e1000_clean_tx_irq - Reclaim resources after transmit completes
3845  * @adapter: board private structure
3846  **/
3847 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
3848                                struct e1000_tx_ring *tx_ring)
3849 {
3850         struct e1000_hw *hw = &adapter->hw;
3851         struct net_device *netdev = adapter->netdev;
3852         struct e1000_tx_desc *tx_desc, *eop_desc;
3853         struct e1000_tx_buffer *buffer_info;
3854         unsigned int i, eop;
3855         unsigned int count = 0;
3856         unsigned int total_tx_bytes = 0, total_tx_packets = 0;
3857         unsigned int bytes_compl = 0, pkts_compl = 0;
3858
3859         i = tx_ring->next_to_clean;
3860         eop = tx_ring->buffer_info[i].next_to_watch;
3861         eop_desc = E1000_TX_DESC(*tx_ring, eop);
3862
3863         while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
3864                (count < tx_ring->count)) {
3865                 bool cleaned = false;
3866                 dma_rmb();      /* read buffer_info after eop_desc */
3867                 for ( ; !cleaned; count++) {
3868                         tx_desc = E1000_TX_DESC(*tx_ring, i);
3869                         buffer_info = &tx_ring->buffer_info[i];
3870                         cleaned = (i == eop);
3871
3872                         if (cleaned) {
3873                                 total_tx_packets += buffer_info->segs;
3874                                 total_tx_bytes += buffer_info->bytecount;
3875                                 if (buffer_info->skb) {
3876                                         bytes_compl += buffer_info->skb->len;
3877                                         pkts_compl++;
3878                                 }
3879
3880                         }
3881                         e1000_unmap_and_free_tx_resource(adapter, buffer_info);
3882                         tx_desc->upper.data = 0;
3883
3884                         if (unlikely(++i == tx_ring->count))
3885                                 i = 0;
3886                 }
3887
3888                 eop = tx_ring->buffer_info[i].next_to_watch;
3889                 eop_desc = E1000_TX_DESC(*tx_ring, eop);
3890         }
3891
3892         /* Synchronize with E1000_DESC_UNUSED called from e1000_xmit_frame,
3893          * which will reuse the cleaned buffers.
3894          */
3895         smp_store_release(&tx_ring->next_to_clean, i);
3896
3897         netdev_completed_queue(netdev, pkts_compl, bytes_compl);
3898
3899 #define TX_WAKE_THRESHOLD 32
3900         if (unlikely(count && netif_carrier_ok(netdev) &&
3901                      E1000_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD)) {
3902                 /* Make sure that anybody stopping the queue after this
3903                  * sees the new next_to_clean.
3904                  */
3905                 smp_mb();
3906
3907                 if (netif_queue_stopped(netdev) &&
3908                     !(test_bit(__E1000_DOWN, &adapter->flags))) {
3909                         netif_wake_queue(netdev);
3910                         ++adapter->restart_queue;
3911                 }
3912         }
3913
3914         if (adapter->detect_tx_hung) {
3915                 /* Detect a transmit hang in hardware, this serializes the
3916                  * check with the clearing of time_stamp and movement of i
3917                  */
3918                 adapter->detect_tx_hung = false;
3919                 if (tx_ring->buffer_info[eop].time_stamp &&
3920                     time_after(jiffies, tx_ring->buffer_info[eop].time_stamp +
3921                                (adapter->tx_timeout_factor * HZ)) &&
3922                     !(er32(STATUS) & E1000_STATUS_TXOFF)) {
3923
3924                         /* detected Tx unit hang */
3925                         e_err(drv, "Detected Tx Unit Hang\n"
3926                               "  Tx Queue             <%lu>\n"
3927                               "  TDH                  <%x>\n"
3928                               "  TDT                  <%x>\n"
3929                               "  next_to_use          <%x>\n"
3930                               "  next_to_clean        <%x>\n"
3931                               "buffer_info[next_to_clean]\n"
3932                               "  time_stamp           <%lx>\n"
3933                               "  next_to_watch        <%x>\n"
3934                               "  jiffies              <%lx>\n"
3935                               "  next_to_watch.status <%x>\n",
3936                                 (unsigned long)(tx_ring - adapter->tx_ring),
3937                                 readl(hw->hw_addr + tx_ring->tdh),
3938                                 readl(hw->hw_addr + tx_ring->tdt),
3939                                 tx_ring->next_to_use,
3940                                 tx_ring->next_to_clean,
3941                                 tx_ring->buffer_info[eop].time_stamp,
3942                                 eop,
3943                                 jiffies,
3944                                 eop_desc->upper.fields.status);
3945                         e1000_dump(adapter);
3946                         netif_stop_queue(netdev);
3947                 }
3948         }
3949         adapter->total_tx_bytes += total_tx_bytes;
3950         adapter->total_tx_packets += total_tx_packets;
3951         netdev->stats.tx_bytes += total_tx_bytes;
3952         netdev->stats.tx_packets += total_tx_packets;
3953         return count < tx_ring->count;
3954 }
3955
3956 /**
3957  * e1000_rx_checksum - Receive Checksum Offload for 82543
3958  * @adapter:     board private structure
3959  * @status_err:  receive descriptor status and error fields
3960  * @csum:        receive descriptor csum field
3961  * @sk_buff:     socket buffer with received data
3962  **/
3963 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
3964                               u32 csum, struct sk_buff *skb)
3965 {
3966         struct e1000_hw *hw = &adapter->hw;
3967         u16 status = (u16)status_err;
3968         u8 errors = (u8)(status_err >> 24);
3969
3970         skb_checksum_none_assert(skb);
3971
3972         /* 82543 or newer only */
3973         if (unlikely(hw->mac_type < e1000_82543))
3974                 return;
3975         /* Ignore Checksum bit is set */
3976         if (unlikely(status & E1000_RXD_STAT_IXSM))
3977                 return;
3978         /* TCP/UDP checksum error bit is set */
3979         if (unlikely(errors & E1000_RXD_ERR_TCPE)) {
3980                 /* let the stack verify checksum errors */
3981                 adapter->hw_csum_err++;
3982                 return;
3983         }
3984         /* TCP/UDP Checksum has not been calculated */
3985         if (!(status & E1000_RXD_STAT_TCPCS))
3986                 return;
3987
3988         /* It must be a TCP or UDP packet with a valid checksum */
3989         if (likely(status & E1000_RXD_STAT_TCPCS)) {
3990                 /* TCP checksum is good */
3991                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3992         }
3993         adapter->hw_csum_good++;
3994 }
3995
3996 /**
3997  * e1000_consume_page - helper function for jumbo Rx path
3998  **/
3999 static void e1000_consume_page(struct e1000_rx_buffer *bi, struct sk_buff *skb,
4000                                u16 length)
4001 {
4002         bi->rxbuf.page = NULL;
4003         skb->len += length;
4004         skb->data_len += length;
4005         skb->truesize += PAGE_SIZE;
4006 }
4007
4008 /**
4009  * e1000_receive_skb - helper function to handle rx indications
4010  * @adapter: board private structure
4011  * @status: descriptor status field as written by hardware
4012  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
4013  * @skb: pointer to sk_buff to be indicated to stack
4014  */
4015 static void e1000_receive_skb(struct e1000_adapter *adapter, u8 status,
4016                               __le16 vlan, struct sk_buff *skb)
4017 {
4018         skb->protocol = eth_type_trans(skb, adapter->netdev);
4019
4020         if (status & E1000_RXD_STAT_VP) {
4021                 u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4022
4023                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4024         }
4025         napi_gro_receive(&adapter->napi, skb);
4026 }
4027
4028 /**
4029  * e1000_tbi_adjust_stats
4030  * @hw: Struct containing variables accessed by shared code
4031  * @frame_len: The length of the frame in question
4032  * @mac_addr: The Ethernet destination address of the frame in question
4033  *
4034  * Adjusts the statistic counters when a frame is accepted by TBI_ACCEPT
4035  */
4036 static void e1000_tbi_adjust_stats(struct e1000_hw *hw,
4037                                    struct e1000_hw_stats *stats,
4038                                    u32 frame_len, const u8 *mac_addr)
4039 {
4040         u64 carry_bit;
4041
4042         /* First adjust the frame length. */
4043         frame_len--;
4044         /* We need to adjust the statistics counters, since the hardware
4045          * counters overcount this packet as a CRC error and undercount
4046          * the packet as a good packet
4047          */
4048         /* This packet should not be counted as a CRC error. */
4049         stats->crcerrs--;
4050         /* This packet does count as a Good Packet Received. */
4051         stats->gprc++;
4052
4053         /* Adjust the Good Octets received counters */
4054         carry_bit = 0x80000000 & stats->gorcl;
4055         stats->gorcl += frame_len;
4056         /* If the high bit of Gorcl (the low 32 bits of the Good Octets
4057          * Received Count) was one before the addition,
4058          * AND it is zero after, then we lost the carry out,
4059          * need to add one to Gorch (Good Octets Received Count High).
4060          * This could be simplified if all environments supported
4061          * 64-bit integers.
4062          */
4063         if (carry_bit && ((stats->gorcl & 0x80000000) == 0))
4064                 stats->gorch++;
4065         /* Is this a broadcast or multicast?  Check broadcast first,
4066          * since the test for a multicast frame will test positive on
4067          * a broadcast frame.
4068          */
4069         if (is_broadcast_ether_addr(mac_addr))
4070                 stats->bprc++;
4071         else if (is_multicast_ether_addr(mac_addr))
4072                 stats->mprc++;
4073
4074         if (frame_len == hw->max_frame_size) {
4075                 /* In this case, the hardware has overcounted the number of
4076                  * oversize frames.
4077                  */
4078                 if (stats->roc > 0)
4079                         stats->roc--;
4080         }
4081
4082         /* Adjust the bin counters when the extra byte put the frame in the
4083          * wrong bin. Remember that the frame_len was adjusted above.
4084          */
4085         if (frame_len == 64) {
4086                 stats->prc64++;
4087                 stats->prc127--;
4088         } else if (frame_len == 127) {
4089                 stats->prc127++;
4090                 stats->prc255--;
4091         } else if (frame_len == 255) {
4092                 stats->prc255++;
4093                 stats->prc511--;
4094         } else if (frame_len == 511) {
4095                 stats->prc511++;
4096                 stats->prc1023--;
4097         } else if (frame_len == 1023) {
4098                 stats->prc1023++;
4099                 stats->prc1522--;
4100         } else if (frame_len == 1522) {
4101                 stats->prc1522++;
4102         }
4103 }
4104
4105 static bool e1000_tbi_should_accept(struct e1000_adapter *adapter,
4106                                     u8 status, u8 errors,
4107                                     u32 length, const u8 *data)
4108 {
4109         struct e1000_hw *hw = &adapter->hw;
4110         u8 last_byte = *(data + length - 1);
4111
4112         if (TBI_ACCEPT(hw, status, errors, length, last_byte)) {
4113                 unsigned long irq_flags;
4114
4115                 spin_lock_irqsave(&adapter->stats_lock, irq_flags);
4116                 e1000_tbi_adjust_stats(hw, &adapter->stats, length, data);
4117                 spin_unlock_irqrestore(&adapter->stats_lock, irq_flags);
4118
4119                 return true;
4120         }
4121
4122         return false;
4123 }
4124
4125 static struct sk_buff *e1000_alloc_rx_skb(struct e1000_adapter *adapter,
4126                                           unsigned int bufsz)
4127 {
4128         struct sk_buff *skb = napi_alloc_skb(&adapter->napi, bufsz);
4129
4130         if (unlikely(!skb))
4131                 adapter->alloc_rx_buff_failed++;
4132         return skb;
4133 }
4134
4135 /**
4136  * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
4137  * @adapter: board private structure
4138  * @rx_ring: ring to clean
4139  * @work_done: amount of napi work completed this call
4140  * @work_to_do: max amount of work allowed for this call to do
4141  *
4142  * the return value indicates whether actual cleaning was done, there
4143  * is no guarantee that everything was cleaned
4144  */
4145 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
4146                                      struct e1000_rx_ring *rx_ring,
4147                                      int *work_done, int work_to_do)
4148 {
4149         struct net_device *netdev = adapter->netdev;
4150         struct pci_dev *pdev = adapter->pdev;
4151         struct e1000_rx_desc *rx_desc, *next_rxd;
4152         struct e1000_rx_buffer *buffer_info, *next_buffer;
4153         u32 length;
4154         unsigned int i;
4155         int cleaned_count = 0;
4156         bool cleaned = false;
4157         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4158
4159         i = rx_ring->next_to_clean;
4160         rx_desc = E1000_RX_DESC(*rx_ring, i);
4161         buffer_info = &rx_ring->buffer_info[i];
4162
4163         while (rx_desc->status & E1000_RXD_STAT_DD) {
4164                 struct sk_buff *skb;
4165                 u8 status;
4166
4167                 if (*work_done >= work_to_do)
4168                         break;
4169                 (*work_done)++;
4170                 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4171
4172                 status = rx_desc->status;
4173
4174                 if (++i == rx_ring->count)
4175                         i = 0;
4176
4177                 next_rxd = E1000_RX_DESC(*rx_ring, i);
4178                 prefetch(next_rxd);
4179
4180                 next_buffer = &rx_ring->buffer_info[i];
4181
4182                 cleaned = true;
4183                 cleaned_count++;
4184                 dma_unmap_page(&pdev->dev, buffer_info->dma,
4185                                adapter->rx_buffer_len, DMA_FROM_DEVICE);
4186                 buffer_info->dma = 0;
4187
4188                 length = le16_to_cpu(rx_desc->length);
4189
4190                 /* errors is only valid for DD + EOP descriptors */
4191                 if (unlikely((status & E1000_RXD_STAT_EOP) &&
4192                     (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK))) {
4193                         u8 *mapped = page_address(buffer_info->rxbuf.page);
4194
4195                         if (e1000_tbi_should_accept(adapter, status,
4196                                                     rx_desc->errors,
4197                                                     length, mapped)) {
4198                                 length--;
4199                         } else if (netdev->features & NETIF_F_RXALL) {
4200                                 goto process_skb;
4201                         } else {
4202                                 /* an error means any chain goes out the window
4203                                  * too
4204                                  */
4205                                 if (rx_ring->rx_skb_top)
4206                                         dev_kfree_skb(rx_ring->rx_skb_top);
4207                                 rx_ring->rx_skb_top = NULL;
4208                                 goto next_desc;
4209                         }
4210                 }
4211
4212 #define rxtop rx_ring->rx_skb_top
4213 process_skb:
4214                 if (!(status & E1000_RXD_STAT_EOP)) {
4215                         /* this descriptor is only the beginning (or middle) */
4216                         if (!rxtop) {
4217                                 /* this is the beginning of a chain */
4218                                 rxtop = napi_get_frags(&adapter->napi);
4219                                 if (!rxtop)
4220                                         break;
4221
4222                                 skb_fill_page_desc(rxtop, 0,
4223                                                    buffer_info->rxbuf.page,
4224                                                    0, length);
4225                         } else {
4226                                 /* this is the middle of a chain */
4227                                 skb_fill_page_desc(rxtop,
4228                                     skb_shinfo(rxtop)->nr_frags,
4229                                     buffer_info->rxbuf.page, 0, length);
4230                         }
4231                         e1000_consume_page(buffer_info, rxtop, length);
4232                         goto next_desc;
4233                 } else {
4234                         if (rxtop) {
4235                                 /* end of the chain */
4236                                 skb_fill_page_desc(rxtop,
4237                                     skb_shinfo(rxtop)->nr_frags,
4238                                     buffer_info->rxbuf.page, 0, length);
4239                                 skb = rxtop;
4240                                 rxtop = NULL;
4241                                 e1000_consume_page(buffer_info, skb, length);
4242                         } else {
4243                                 struct page *p;
4244                                 /* no chain, got EOP, this buf is the packet
4245                                  * copybreak to save the put_page/alloc_page
4246                                  */
4247                                 p = buffer_info->rxbuf.page;
4248                                 if (length <= copybreak) {
4249                                         u8 *vaddr;
4250
4251                                         if (likely(!(netdev->features & NETIF_F_RXFCS)))
4252                                                 length -= 4;
4253                                         skb = e1000_alloc_rx_skb(adapter,
4254                                                                  length);
4255                                         if (!skb)
4256                                                 break;
4257
4258                                         vaddr = kmap_atomic(p);
4259                                         memcpy(skb_tail_pointer(skb), vaddr,
4260                                                length);
4261                                         kunmap_atomic(vaddr);
4262                                         /* re-use the page, so don't erase
4263                                          * buffer_info->rxbuf.page
4264                                          */
4265                                         skb_put(skb, length);
4266                                         e1000_rx_checksum(adapter,
4267                                                           status | rx_desc->errors << 24,
4268                                                           le16_to_cpu(rx_desc->csum), skb);
4269
4270                                         total_rx_bytes += skb->len;
4271                                         total_rx_packets++;
4272
4273                                         e1000_receive_skb(adapter, status,
4274                                                           rx_desc->special, skb);
4275                                         goto next_desc;
4276                                 } else {
4277                                         skb = napi_get_frags(&adapter->napi);
4278                                         if (!skb) {
4279                                                 adapter->alloc_rx_buff_failed++;
4280                                                 break;
4281                                         }
4282                                         skb_fill_page_desc(skb, 0, p, 0,
4283                                                            length);
4284                                         e1000_consume_page(buffer_info, skb,
4285                                                            length);
4286                                 }
4287                         }
4288                 }
4289
4290                 /* Receive Checksum Offload XXX recompute due to CRC strip? */
4291                 e1000_rx_checksum(adapter,
4292                                   (u32)(status) |
4293                                   ((u32)(rx_desc->errors) << 24),
4294                                   le16_to_cpu(rx_desc->csum), skb);
4295
4296                 total_rx_bytes += (skb->len - 4); /* don't count FCS */
4297                 if (likely(!(netdev->features & NETIF_F_RXFCS)))
4298                         pskb_trim(skb, skb->len - 4);
4299                 total_rx_packets++;
4300
4301                 if (status & E1000_RXD_STAT_VP) {
4302                         __le16 vlan = rx_desc->special;
4303                         u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4304
4305                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4306                 }
4307
4308                 napi_gro_frags(&adapter->napi);
4309
4310 next_desc:
4311                 rx_desc->status = 0;
4312
4313                 /* return some buffers to hardware, one at a time is too slow */
4314                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4315                         adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4316                         cleaned_count = 0;
4317                 }
4318
4319                 /* use prefetched values */
4320                 rx_desc = next_rxd;
4321                 buffer_info = next_buffer;
4322         }
4323         rx_ring->next_to_clean = i;
4324
4325         cleaned_count = E1000_DESC_UNUSED(rx_ring);
4326         if (cleaned_count)
4327                 adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4328
4329         adapter->total_rx_packets += total_rx_packets;
4330         adapter->total_rx_bytes += total_rx_bytes;
4331         netdev->stats.rx_bytes += total_rx_bytes;
4332         netdev->stats.rx_packets += total_rx_packets;
4333         return cleaned;
4334 }
4335
4336 /* this should improve performance for small packets with large amounts
4337  * of reassembly being done in the stack
4338  */
4339 static struct sk_buff *e1000_copybreak(struct e1000_adapter *adapter,
4340                                        struct e1000_rx_buffer *buffer_info,
4341                                        u32 length, const void *data)
4342 {
4343         struct sk_buff *skb;
4344
4345         if (length > copybreak)
4346                 return NULL;
4347
4348         skb = e1000_alloc_rx_skb(adapter, length);
4349         if (!skb)
4350                 return NULL;
4351
4352         dma_sync_single_for_cpu(&adapter->pdev->dev, buffer_info->dma,
4353                                 length, DMA_FROM_DEVICE);
4354
4355         skb_put_data(skb, data, length);
4356
4357         return skb;
4358 }
4359
4360 /**
4361  * e1000_clean_rx_irq - Send received data up the network stack; legacy
4362  * @adapter: board private structure
4363  * @rx_ring: ring to clean
4364  * @work_done: amount of napi work completed this call
4365  * @work_to_do: max amount of work allowed for this call to do
4366  */
4367 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
4368                                struct e1000_rx_ring *rx_ring,
4369                                int *work_done, int work_to_do)
4370 {
4371         struct net_device *netdev = adapter->netdev;
4372         struct pci_dev *pdev = adapter->pdev;
4373         struct e1000_rx_desc *rx_desc, *next_rxd;
4374         struct e1000_rx_buffer *buffer_info, *next_buffer;
4375         u32 length;
4376         unsigned int i;
4377         int cleaned_count = 0;
4378         bool cleaned = false;
4379         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4380
4381         i = rx_ring->next_to_clean;
4382         rx_desc = E1000_RX_DESC(*rx_ring, i);
4383         buffer_info = &rx_ring->buffer_info[i];
4384
4385         while (rx_desc->status & E1000_RXD_STAT_DD) {
4386                 struct sk_buff *skb;
4387                 u8 *data;
4388                 u8 status;
4389
4390                 if (*work_done >= work_to_do)
4391                         break;
4392                 (*work_done)++;
4393                 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4394
4395                 status = rx_desc->status;
4396                 length = le16_to_cpu(rx_desc->length);
4397
4398                 data = buffer_info->rxbuf.data;
4399                 prefetch(data);
4400                 skb = e1000_copybreak(adapter, buffer_info, length, data);
4401                 if (!skb) {
4402                         unsigned int frag_len = e1000_frag_len(adapter);
4403
4404                         skb = build_skb(data - E1000_HEADROOM, frag_len);
4405                         if (!skb) {
4406                                 adapter->alloc_rx_buff_failed++;
4407                                 break;
4408                         }
4409
4410                         skb_reserve(skb, E1000_HEADROOM);
4411                         dma_unmap_single(&pdev->dev, buffer_info->dma,
4412                                          adapter->rx_buffer_len,
4413                                          DMA_FROM_DEVICE);
4414                         buffer_info->dma = 0;
4415                         buffer_info->rxbuf.data = NULL;
4416                 }
4417
4418                 if (++i == rx_ring->count)
4419                         i = 0;
4420
4421                 next_rxd = E1000_RX_DESC(*rx_ring, i);
4422                 prefetch(next_rxd);
4423
4424                 next_buffer = &rx_ring->buffer_info[i];
4425
4426                 cleaned = true;
4427                 cleaned_count++;
4428
4429                 /* !EOP means multiple descriptors were used to store a single
4430                  * packet, if thats the case we need to toss it.  In fact, we
4431                  * to toss every packet with the EOP bit clear and the next
4432                  * frame that _does_ have the EOP bit set, as it is by
4433                  * definition only a frame fragment
4434                  */
4435                 if (unlikely(!(status & E1000_RXD_STAT_EOP)))
4436                         adapter->discarding = true;
4437
4438                 if (adapter->discarding) {
4439                         /* All receives must fit into a single buffer */
4440                         netdev_dbg(netdev, "Receive packet consumed multiple buffers\n");
4441                         dev_kfree_skb(skb);
4442                         if (status & E1000_RXD_STAT_EOP)
4443                                 adapter->discarding = false;
4444                         goto next_desc;
4445                 }
4446
4447                 if (unlikely(rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK)) {
4448                         if (e1000_tbi_should_accept(adapter, status,
4449                                                     rx_desc->errors,
4450                                                     length, data)) {
4451                                 length--;
4452                         } else if (netdev->features & NETIF_F_RXALL) {
4453                                 goto process_skb;
4454                         } else {
4455                                 dev_kfree_skb(skb);
4456                                 goto next_desc;
4457                         }
4458                 }
4459
4460 process_skb:
4461                 total_rx_bytes += (length - 4); /* don't count FCS */
4462                 total_rx_packets++;
4463
4464                 if (likely(!(netdev->features & NETIF_F_RXFCS)))
4465                         /* adjust length to remove Ethernet CRC, this must be
4466                          * done after the TBI_ACCEPT workaround above
4467                          */
4468                         length -= 4;
4469
4470                 if (buffer_info->rxbuf.data == NULL)
4471                         skb_put(skb, length);
4472                 else /* copybreak skb */
4473                         skb_trim(skb, length);
4474
4475                 /* Receive Checksum Offload */
4476                 e1000_rx_checksum(adapter,
4477                                   (u32)(status) |
4478                                   ((u32)(rx_desc->errors) << 24),
4479                                   le16_to_cpu(rx_desc->csum), skb);
4480
4481                 e1000_receive_skb(adapter, status, rx_desc->special, skb);
4482
4483 next_desc:
4484                 rx_desc->status = 0;
4485
4486                 /* return some buffers to hardware, one at a time is too slow */
4487                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4488                         adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4489                         cleaned_count = 0;
4490                 }
4491
4492                 /* use prefetched values */
4493                 rx_desc = next_rxd;
4494                 buffer_info = next_buffer;
4495         }
4496         rx_ring->next_to_clean = i;
4497
4498         cleaned_count = E1000_DESC_UNUSED(rx_ring);
4499         if (cleaned_count)
4500                 adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4501
4502         adapter->total_rx_packets += total_rx_packets;
4503         adapter->total_rx_bytes += total_rx_bytes;
4504         netdev->stats.rx_bytes += total_rx_bytes;
4505         netdev->stats.rx_packets += total_rx_packets;
4506         return cleaned;
4507 }
4508
4509 /**
4510  * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
4511  * @adapter: address of board private structure
4512  * @rx_ring: pointer to receive ring structure
4513  * @cleaned_count: number of buffers to allocate this pass
4514  **/
4515 static void
4516 e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
4517                              struct e1000_rx_ring *rx_ring, int cleaned_count)
4518 {
4519         struct pci_dev *pdev = adapter->pdev;
4520         struct e1000_rx_desc *rx_desc;
4521         struct e1000_rx_buffer *buffer_info;
4522         unsigned int i;
4523
4524         i = rx_ring->next_to_use;
4525         buffer_info = &rx_ring->buffer_info[i];
4526
4527         while (cleaned_count--) {
4528                 /* allocate a new page if necessary */
4529                 if (!buffer_info->rxbuf.page) {
4530                         buffer_info->rxbuf.page = alloc_page(GFP_ATOMIC);
4531                         if (unlikely(!buffer_info->rxbuf.page)) {
4532                                 adapter->alloc_rx_buff_failed++;
4533                                 break;
4534                         }
4535                 }
4536
4537                 if (!buffer_info->dma) {
4538                         buffer_info->dma = dma_map_page(&pdev->dev,
4539                                                         buffer_info->rxbuf.page, 0,
4540                                                         adapter->rx_buffer_len,
4541                                                         DMA_FROM_DEVICE);
4542                         if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4543                                 put_page(buffer_info->rxbuf.page);
4544                                 buffer_info->rxbuf.page = NULL;
4545                                 buffer_info->dma = 0;
4546                                 adapter->alloc_rx_buff_failed++;
4547                                 break;
4548                         }
4549                 }
4550
4551                 rx_desc = E1000_RX_DESC(*rx_ring, i);
4552                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4553
4554                 if (unlikely(++i == rx_ring->count))
4555                         i = 0;
4556                 buffer_info = &rx_ring->buffer_info[i];
4557         }
4558
4559         if (likely(rx_ring->next_to_use != i)) {
4560                 rx_ring->next_to_use = i;
4561                 if (unlikely(i-- == 0))
4562                         i = (rx_ring->count - 1);
4563
4564                 /* Force memory writes to complete before letting h/w
4565                  * know there are new descriptors to fetch.  (Only
4566                  * applicable for weak-ordered memory model archs,
4567                  * such as IA-64).
4568                  */
4569                 wmb();
4570                 writel(i, adapter->hw.hw_addr + rx_ring->rdt);
4571         }
4572 }
4573
4574 /**
4575  * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended
4576  * @adapter: address of board private structure
4577  **/
4578 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
4579                                    struct e1000_rx_ring *rx_ring,
4580                                    int cleaned_count)
4581 {
4582         struct e1000_hw *hw = &adapter->hw;
4583         struct pci_dev *pdev = adapter->pdev;
4584         struct e1000_rx_desc *rx_desc;
4585         struct e1000_rx_buffer *buffer_info;
4586         unsigned int i;
4587         unsigned int bufsz = adapter->rx_buffer_len;
4588
4589         i = rx_ring->next_to_use;
4590         buffer_info = &rx_ring->buffer_info[i];
4591
4592         while (cleaned_count--) {
4593                 void *data;
4594
4595                 if (buffer_info->rxbuf.data)
4596                         goto skip;
4597
4598                 data = e1000_alloc_frag(adapter);
4599                 if (!data) {
4600                         /* Better luck next round */
4601                         adapter->alloc_rx_buff_failed++;
4602                         break;
4603                 }
4604
4605                 /* Fix for errata 23, can't cross 64kB boundary */
4606                 if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4607                         void *olddata = data;
4608                         e_err(rx_err, "skb align check failed: %u bytes at "
4609                               "%p\n", bufsz, data);
4610                         /* Try again, without freeing the previous */
4611                         data = e1000_alloc_frag(adapter);
4612                         /* Failed allocation, critical failure */
4613                         if (!data) {
4614                                 skb_free_frag(olddata);
4615                                 adapter->alloc_rx_buff_failed++;
4616                                 break;
4617                         }
4618
4619                         if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4620                                 /* give up */
4621                                 skb_free_frag(data);
4622                                 skb_free_frag(olddata);
4623                                 adapter->alloc_rx_buff_failed++;
4624                                 break;
4625                         }
4626
4627                         /* Use new allocation */
4628                         skb_free_frag(olddata);
4629                 }
4630                 buffer_info->dma = dma_map_single(&pdev->dev,
4631                                                   data,
4632                                                   adapter->rx_buffer_len,
4633                                                   DMA_FROM_DEVICE);
4634                 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4635                         skb_free_frag(data);
4636                         buffer_info->dma = 0;
4637                         adapter->alloc_rx_buff_failed++;
4638                         break;
4639                 }
4640
4641                 /* XXX if it was allocated cleanly it will never map to a
4642                  * boundary crossing
4643                  */
4644
4645                 /* Fix for errata 23, can't cross 64kB boundary */
4646                 if (!e1000_check_64k_bound(adapter,
4647                                         (void *)(unsigned long)buffer_info->dma,
4648                                         adapter->rx_buffer_len)) {
4649                         e_err(rx_err, "dma align check failed: %u bytes at "
4650                               "%p\n", adapter->rx_buffer_len,
4651                               (void *)(unsigned long)buffer_info->dma);
4652
4653                         dma_unmap_single(&pdev->dev, buffer_info->dma,
4654                                          adapter->rx_buffer_len,
4655                                          DMA_FROM_DEVICE);
4656
4657                         skb_free_frag(data);
4658                         buffer_info->rxbuf.data = NULL;
4659                         buffer_info->dma = 0;
4660
4661                         adapter->alloc_rx_buff_failed++;
4662                         break;
4663                 }
4664                 buffer_info->rxbuf.data = data;
4665  skip:
4666                 rx_desc = E1000_RX_DESC(*rx_ring, i);
4667                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4668
4669                 if (unlikely(++i == rx_ring->count))
4670                         i = 0;
4671                 buffer_info = &rx_ring->buffer_info[i];
4672         }
4673
4674         if (likely(rx_ring->next_to_use != i)) {
4675                 rx_ring->next_to_use = i;
4676                 if (unlikely(i-- == 0))
4677                         i = (rx_ring->count - 1);
4678
4679                 /* Force memory writes to complete before letting h/w
4680                  * know there are new descriptors to fetch.  (Only
4681                  * applicable for weak-ordered memory model archs,
4682                  * such as IA-64).
4683                  */
4684                 wmb();
4685                 writel(i, hw->hw_addr + rx_ring->rdt);
4686         }
4687 }
4688
4689 /**
4690  * e1000_smartspeed - Workaround for SmartSpeed on 82541 and 82547 controllers.
4691  * @adapter:
4692  **/
4693 static void e1000_smartspeed(struct e1000_adapter *adapter)
4694 {
4695         struct e1000_hw *hw = &adapter->hw;
4696         u16 phy_status;
4697         u16 phy_ctrl;
4698
4699         if ((hw->phy_type != e1000_phy_igp) || !hw->autoneg ||
4700            !(hw->autoneg_advertised & ADVERTISE_1000_FULL))
4701                 return;
4702
4703         if (adapter->smartspeed == 0) {
4704                 /* If Master/Slave config fault is asserted twice,
4705                  * we assume back-to-back
4706                  */
4707                 e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4708                 if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4709                         return;
4710                 e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4711                 if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4712                         return;
4713                 e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4714                 if (phy_ctrl & CR_1000T_MS_ENABLE) {
4715                         phy_ctrl &= ~CR_1000T_MS_ENABLE;
4716                         e1000_write_phy_reg(hw, PHY_1000T_CTRL,
4717                                             phy_ctrl);
4718                         adapter->smartspeed++;
4719                         if (!e1000_phy_setup_autoneg(hw) &&
4720                            !e1000_read_phy_reg(hw, PHY_CTRL,
4721                                                &phy_ctrl)) {
4722                                 phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4723                                              MII_CR_RESTART_AUTO_NEG);
4724                                 e1000_write_phy_reg(hw, PHY_CTRL,
4725                                                     phy_ctrl);
4726                         }
4727                 }
4728                 return;
4729         } else if (adapter->smartspeed == E1000_SMARTSPEED_DOWNSHIFT) {
4730                 /* If still no link, perhaps using 2/3 pair cable */
4731                 e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4732                 phy_ctrl |= CR_1000T_MS_ENABLE;
4733                 e1000_write_phy_reg(hw, PHY_1000T_CTRL, phy_ctrl);
4734                 if (!e1000_phy_setup_autoneg(hw) &&
4735                    !e1000_read_phy_reg(hw, PHY_CTRL, &phy_ctrl)) {
4736                         phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4737                                      MII_CR_RESTART_AUTO_NEG);
4738                         e1000_write_phy_reg(hw, PHY_CTRL, phy_ctrl);
4739                 }
4740         }
4741         /* Restart process after E1000_SMARTSPEED_MAX iterations */
4742         if (adapter->smartspeed++ == E1000_SMARTSPEED_MAX)
4743                 adapter->smartspeed = 0;
4744 }
4745
4746 /**
4747  * e1000_ioctl -
4748  * @netdev:
4749  * @ifreq:
4750  * @cmd:
4751  **/
4752 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
4753 {
4754         switch (cmd) {
4755         case SIOCGMIIPHY:
4756         case SIOCGMIIREG:
4757         case SIOCSMIIREG:
4758                 return e1000_mii_ioctl(netdev, ifr, cmd);
4759         default:
4760                 return -EOPNOTSUPP;
4761         }
4762 }
4763
4764 /**
4765  * e1000_mii_ioctl -
4766  * @netdev:
4767  * @ifreq:
4768  * @cmd:
4769  **/
4770 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
4771                            int cmd)
4772 {
4773         struct e1000_adapter *adapter = netdev_priv(netdev);
4774         struct e1000_hw *hw = &adapter->hw;
4775         struct mii_ioctl_data *data = if_mii(ifr);
4776         int retval;
4777         u16 mii_reg;
4778         unsigned long flags;
4779
4780         if (hw->media_type != e1000_media_type_copper)
4781                 return -EOPNOTSUPP;
4782
4783         switch (cmd) {
4784         case SIOCGMIIPHY:
4785                 data->phy_id = hw->phy_addr;
4786                 break;
4787         case SIOCGMIIREG:
4788                 spin_lock_irqsave(&adapter->stats_lock, flags);
4789                 if (e1000_read_phy_reg(hw, data->reg_num & 0x1F,
4790                                    &data->val_out)) {
4791                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4792                         return -EIO;
4793                 }
4794                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4795                 break;
4796         case SIOCSMIIREG:
4797                 if (data->reg_num & ~(0x1F))
4798                         return -EFAULT;
4799                 mii_reg = data->val_in;
4800                 spin_lock_irqsave(&adapter->stats_lock, flags);
4801                 if (e1000_write_phy_reg(hw, data->reg_num,
4802                                         mii_reg)) {
4803                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4804                         return -EIO;
4805                 }
4806                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4807                 if (hw->media_type == e1000_media_type_copper) {
4808                         switch (data->reg_num) {
4809                         case PHY_CTRL:
4810                                 if (mii_reg & MII_CR_POWER_DOWN)
4811                                         break;
4812                                 if (mii_reg & MII_CR_AUTO_NEG_EN) {
4813                                         hw->autoneg = 1;
4814                                         hw->autoneg_advertised = 0x2F;
4815                                 } else {
4816                                         u32 speed;
4817                                         if (mii_reg & 0x40)
4818                                                 speed = SPEED_1000;
4819                                         else if (mii_reg & 0x2000)
4820                                                 speed = SPEED_100;
4821                                         else
4822                                                 speed = SPEED_10;
4823                                         retval = e1000_set_spd_dplx(
4824                                                 adapter, speed,
4825                                                 ((mii_reg & 0x100)
4826                                                  ? DUPLEX_FULL :
4827                                                  DUPLEX_HALF));
4828                                         if (retval)
4829                                                 return retval;
4830                                 }
4831                                 if (netif_running(adapter->netdev))
4832                                         e1000_reinit_locked(adapter);
4833                                 else
4834                                         e1000_reset(adapter);
4835                                 break;
4836                         case M88E1000_PHY_SPEC_CTRL:
4837                         case M88E1000_EXT_PHY_SPEC_CTRL:
4838                                 if (e1000_phy_reset(hw))
4839                                         return -EIO;
4840                                 break;
4841                         }
4842                 } else {
4843                         switch (data->reg_num) {
4844                         case PHY_CTRL:
4845                                 if (mii_reg & MII_CR_POWER_DOWN)
4846                                         break;
4847                                 if (netif_running(adapter->netdev))
4848                                         e1000_reinit_locked(adapter);
4849                                 else
4850                                         e1000_reset(adapter);
4851                                 break;
4852                         }
4853                 }
4854                 break;
4855         default:
4856                 return -EOPNOTSUPP;
4857         }
4858         return E1000_SUCCESS;
4859 }
4860
4861 void e1000_pci_set_mwi(struct e1000_hw *hw)
4862 {
4863         struct e1000_adapter *adapter = hw->back;
4864         int ret_val = pci_set_mwi(adapter->pdev);
4865
4866         if (ret_val)
4867                 e_err(probe, "Error in setting MWI\n");
4868 }
4869
4870 void e1000_pci_clear_mwi(struct e1000_hw *hw)
4871 {
4872         struct e1000_adapter *adapter = hw->back;
4873
4874         pci_clear_mwi(adapter->pdev);
4875 }
4876
4877 int e1000_pcix_get_mmrbc(struct e1000_hw *hw)
4878 {
4879         struct e1000_adapter *adapter = hw->back;
4880         return pcix_get_mmrbc(adapter->pdev);
4881 }
4882
4883 void e1000_pcix_set_mmrbc(struct e1000_hw *hw, int mmrbc)
4884 {
4885         struct e1000_adapter *adapter = hw->back;
4886         pcix_set_mmrbc(adapter->pdev, mmrbc);
4887 }
4888
4889 void e1000_io_write(struct e1000_hw *hw, unsigned long port, u32 value)
4890 {
4891         outl(value, port);
4892 }
4893
4894 static bool e1000_vlan_used(struct e1000_adapter *adapter)
4895 {
4896         u16 vid;
4897
4898         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
4899                 return true;
4900         return false;
4901 }
4902
4903 static void __e1000_vlan_mode(struct e1000_adapter *adapter,
4904                               netdev_features_t features)
4905 {
4906         struct e1000_hw *hw = &adapter->hw;
4907         u32 ctrl;
4908
4909         ctrl = er32(CTRL);
4910         if (features & NETIF_F_HW_VLAN_CTAG_RX) {
4911                 /* enable VLAN tag insert/strip */
4912                 ctrl |= E1000_CTRL_VME;
4913         } else {
4914                 /* disable VLAN tag insert/strip */
4915                 ctrl &= ~E1000_CTRL_VME;
4916         }
4917         ew32(CTRL, ctrl);
4918 }
4919 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
4920                                      bool filter_on)
4921 {
4922         struct e1000_hw *hw = &adapter->hw;
4923         u32 rctl;
4924
4925         if (!test_bit(__E1000_DOWN, &adapter->flags))
4926                 e1000_irq_disable(adapter);
4927
4928         __e1000_vlan_mode(adapter, adapter->netdev->features);
4929         if (filter_on) {
4930                 /* enable VLAN receive filtering */
4931                 rctl = er32(RCTL);
4932                 rctl &= ~E1000_RCTL_CFIEN;
4933                 if (!(adapter->netdev->flags & IFF_PROMISC))
4934                         rctl |= E1000_RCTL_VFE;
4935                 ew32(RCTL, rctl);
4936                 e1000_update_mng_vlan(adapter);
4937         } else {
4938                 /* disable VLAN receive filtering */
4939                 rctl = er32(RCTL);
4940                 rctl &= ~E1000_RCTL_VFE;
4941                 ew32(RCTL, rctl);
4942         }
4943
4944         if (!test_bit(__E1000_DOWN, &adapter->flags))
4945                 e1000_irq_enable(adapter);
4946 }
4947
4948 static void e1000_vlan_mode(struct net_device *netdev,
4949                             netdev_features_t features)
4950 {
4951         struct e1000_adapter *adapter = netdev_priv(netdev);
4952
4953         if (!test_bit(__E1000_DOWN, &adapter->flags))
4954                 e1000_irq_disable(adapter);
4955
4956         __e1000_vlan_mode(adapter, features);
4957
4958         if (!test_bit(__E1000_DOWN, &adapter->flags))
4959                 e1000_irq_enable(adapter);
4960 }
4961
4962 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
4963                                  __be16 proto, u16 vid)
4964 {
4965         struct e1000_adapter *adapter = netdev_priv(netdev);
4966         struct e1000_hw *hw = &adapter->hw;
4967         u32 vfta, index;
4968
4969         if ((hw->mng_cookie.status &
4970              E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
4971             (vid == adapter->mng_vlan_id))
4972                 return 0;
4973
4974         if (!e1000_vlan_used(adapter))
4975                 e1000_vlan_filter_on_off(adapter, true);
4976
4977         /* add VID to filter table */
4978         index = (vid >> 5) & 0x7F;
4979         vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4980         vfta |= (1 << (vid & 0x1F));
4981         e1000_write_vfta(hw, index, vfta);
4982
4983         set_bit(vid, adapter->active_vlans);
4984
4985         return 0;
4986 }
4987
4988 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
4989                                   __be16 proto, u16 vid)
4990 {
4991         struct e1000_adapter *adapter = netdev_priv(netdev);
4992         struct e1000_hw *hw = &adapter->hw;
4993         u32 vfta, index;
4994
4995         if (!test_bit(__E1000_DOWN, &adapter->flags))
4996                 e1000_irq_disable(adapter);
4997         if (!test_bit(__E1000_DOWN, &adapter->flags))
4998                 e1000_irq_enable(adapter);
4999
5000         /* remove VID from filter table */
5001         index = (vid >> 5) & 0x7F;
5002         vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
5003         vfta &= ~(1 << (vid & 0x1F));
5004         e1000_write_vfta(hw, index, vfta);
5005
5006         clear_bit(vid, adapter->active_vlans);
5007
5008         if (!e1000_vlan_used(adapter))
5009                 e1000_vlan_filter_on_off(adapter, false);
5010
5011         return 0;
5012 }
5013
5014 static void e1000_restore_vlan(struct e1000_adapter *adapter)
5015 {
5016         u16 vid;
5017
5018         if (!e1000_vlan_used(adapter))
5019                 return;
5020
5021         e1000_vlan_filter_on_off(adapter, true);
5022         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
5023                 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
5024 }
5025
5026 int e1000_set_spd_dplx(struct e1000_adapter *adapter, u32 spd, u8 dplx)
5027 {
5028         struct e1000_hw *hw = &adapter->hw;
5029
5030         hw->autoneg = 0;
5031
5032         /* Make sure dplx is at most 1 bit and lsb of speed is not set
5033          * for the switch() below to work
5034          */
5035         if ((spd & 1) || (dplx & ~1))
5036                 goto err_inval;
5037
5038         /* Fiber NICs only allow 1000 gbps Full duplex */
5039         if ((hw->media_type == e1000_media_type_fiber) &&
5040             spd != SPEED_1000 &&
5041             dplx != DUPLEX_FULL)
5042                 goto err_inval;
5043
5044         switch (spd + dplx) {
5045         case SPEED_10 + DUPLEX_HALF:
5046                 hw->forced_speed_duplex = e1000_10_half;
5047                 break;
5048         case SPEED_10 + DUPLEX_FULL:
5049                 hw->forced_speed_duplex = e1000_10_full;
5050                 break;
5051         case SPEED_100 + DUPLEX_HALF:
5052                 hw->forced_speed_duplex = e1000_100_half;
5053                 break;
5054         case SPEED_100 + DUPLEX_FULL:
5055                 hw->forced_speed_duplex = e1000_100_full;
5056                 break;
5057         case SPEED_1000 + DUPLEX_FULL:
5058                 hw->autoneg = 1;
5059                 hw->autoneg_advertised = ADVERTISE_1000_FULL;
5060                 break;
5061         case SPEED_1000 + DUPLEX_HALF: /* not supported */
5062         default:
5063                 goto err_inval;
5064         }
5065
5066         /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
5067         hw->mdix = AUTO_ALL_MODES;
5068
5069         return 0;
5070
5071 err_inval:
5072         e_err(probe, "Unsupported Speed/Duplex configuration\n");
5073         return -EINVAL;
5074 }
5075
5076 static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake)
5077 {
5078         struct net_device *netdev = pci_get_drvdata(pdev);
5079         struct e1000_adapter *adapter = netdev_priv(netdev);
5080         struct e1000_hw *hw = &adapter->hw;
5081         u32 ctrl, ctrl_ext, rctl, status;
5082         u32 wufc = adapter->wol;
5083 #ifdef CONFIG_PM
5084         int retval = 0;
5085 #endif
5086
5087         netif_device_detach(netdev);
5088
5089         if (netif_running(netdev)) {
5090                 int count = E1000_CHECK_RESET_COUNT;
5091
5092                 while (test_bit(__E1000_RESETTING, &adapter->flags) && count--)
5093                         usleep_range(10000, 20000);
5094
5095                 WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
5096                 e1000_down(adapter);
5097         }
5098
5099 #ifdef CONFIG_PM
5100         retval = pci_save_state(pdev);
5101         if (retval)
5102                 return retval;
5103 #endif
5104
5105         status = er32(STATUS);
5106         if (status & E1000_STATUS_LU)
5107                 wufc &= ~E1000_WUFC_LNKC;
5108
5109         if (wufc) {
5110                 e1000_setup_rctl(adapter);
5111                 e1000_set_rx_mode(netdev);
5112
5113                 rctl = er32(RCTL);
5114
5115                 /* turn on all-multi mode if wake on multicast is enabled */
5116                 if (wufc & E1000_WUFC_MC)
5117                         rctl |= E1000_RCTL_MPE;
5118
5119                 /* enable receives in the hardware */
5120                 ew32(RCTL, rctl | E1000_RCTL_EN);
5121
5122                 if (hw->mac_type >= e1000_82540) {
5123                         ctrl = er32(CTRL);
5124                         /* advertise wake from D3Cold */
5125                         #define E1000_CTRL_ADVD3WUC 0x00100000
5126                         /* phy power management enable */
5127                         #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5128                         ctrl |= E1000_CTRL_ADVD3WUC |
5129                                 E1000_CTRL_EN_PHY_PWR_MGMT;
5130                         ew32(CTRL, ctrl);
5131                 }
5132
5133                 if (hw->media_type == e1000_media_type_fiber ||
5134                     hw->media_type == e1000_media_type_internal_serdes) {
5135                         /* keep the laser running in D3 */
5136                         ctrl_ext = er32(CTRL_EXT);
5137                         ctrl_ext |= E1000_CTRL_EXT_SDP7_DATA;
5138                         ew32(CTRL_EXT, ctrl_ext);
5139                 }
5140
5141                 ew32(WUC, E1000_WUC_PME_EN);
5142                 ew32(WUFC, wufc);
5143         } else {
5144                 ew32(WUC, 0);
5145                 ew32(WUFC, 0);
5146         }
5147
5148         e1000_release_manageability(adapter);
5149
5150         *enable_wake = !!wufc;
5151
5152         /* make sure adapter isn't asleep if manageability is enabled */
5153         if (adapter->en_mng_pt)
5154                 *enable_wake = true;
5155
5156         if (netif_running(netdev))
5157                 e1000_free_irq(adapter);
5158
5159         pci_disable_device(pdev);
5160
5161         return 0;
5162 }
5163
5164 #ifdef CONFIG_PM
5165 static int e1000_suspend(struct pci_dev *pdev, pm_message_t state)
5166 {
5167         int retval;
5168         bool wake;
5169
5170         retval = __e1000_shutdown(pdev, &wake);
5171         if (retval)
5172                 return retval;
5173
5174         if (wake) {
5175                 pci_prepare_to_sleep(pdev);
5176         } else {
5177                 pci_wake_from_d3(pdev, false);
5178                 pci_set_power_state(pdev, PCI_D3hot);
5179         }
5180
5181         return 0;
5182 }
5183
5184 static int e1000_resume(struct pci_dev *pdev)
5185 {
5186         struct net_device *netdev = pci_get_drvdata(pdev);
5187         struct e1000_adapter *adapter = netdev_priv(netdev);
5188         struct e1000_hw *hw = &adapter->hw;
5189         u32 err;
5190
5191         pci_set_power_state(pdev, PCI_D0);
5192         pci_restore_state(pdev);
5193         pci_save_state(pdev);
5194
5195         if (adapter->need_ioport)
5196                 err = pci_enable_device(pdev);
5197         else
5198                 err = pci_enable_device_mem(pdev);
5199         if (err) {
5200                 pr_err("Cannot enable PCI device from suspend\n");
5201                 return err;
5202         }
5203         pci_set_master(pdev);
5204
5205         pci_enable_wake(pdev, PCI_D3hot, 0);
5206         pci_enable_wake(pdev, PCI_D3cold, 0);
5207
5208         if (netif_running(netdev)) {
5209                 err = e1000_request_irq(adapter);
5210                 if (err)
5211                         return err;
5212         }
5213
5214         e1000_power_up_phy(adapter);
5215         e1000_reset(adapter);
5216         ew32(WUS, ~0);
5217
5218         e1000_init_manageability(adapter);
5219
5220         if (netif_running(netdev))
5221                 e1000_up(adapter);
5222
5223         netif_device_attach(netdev);
5224
5225         return 0;
5226 }
5227 #endif
5228
5229 static void e1000_shutdown(struct pci_dev *pdev)
5230 {
5231         bool wake;
5232
5233         __e1000_shutdown(pdev, &wake);
5234
5235         if (system_state == SYSTEM_POWER_OFF) {
5236                 pci_wake_from_d3(pdev, wake);
5237                 pci_set_power_state(pdev, PCI_D3hot);
5238         }
5239 }
5240
5241 #ifdef CONFIG_NET_POLL_CONTROLLER
5242 /* Polling 'interrupt' - used by things like netconsole to send skbs
5243  * without having to re-enable interrupts. It's not called while
5244  * the interrupt routine is executing.
5245  */
5246 static void e1000_netpoll(struct net_device *netdev)
5247 {
5248         struct e1000_adapter *adapter = netdev_priv(netdev);
5249
5250         if (disable_hardirq(adapter->pdev->irq))
5251                 e1000_intr(adapter->pdev->irq, netdev);
5252         enable_irq(adapter->pdev->irq);
5253 }
5254 #endif
5255
5256 /**
5257  * e1000_io_error_detected - called when PCI error is detected
5258  * @pdev: Pointer to PCI device
5259  * @state: The current pci connection state
5260  *
5261  * This function is called after a PCI bus error affecting
5262  * this device has been detected.
5263  */
5264 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
5265                                                 pci_channel_state_t state)
5266 {
5267         struct net_device *netdev = pci_get_drvdata(pdev);
5268         struct e1000_adapter *adapter = netdev_priv(netdev);
5269
5270         netif_device_detach(netdev);
5271
5272         if (state == pci_channel_io_perm_failure)
5273                 return PCI_ERS_RESULT_DISCONNECT;
5274
5275         if (netif_running(netdev))
5276                 e1000_down(adapter);
5277         pci_disable_device(pdev);
5278
5279         /* Request a slot slot reset. */
5280         return PCI_ERS_RESULT_NEED_RESET;
5281 }
5282
5283 /**
5284  * e1000_io_slot_reset - called after the pci bus has been reset.
5285  * @pdev: Pointer to PCI device
5286  *
5287  * Restart the card from scratch, as if from a cold-boot. Implementation
5288  * resembles the first-half of the e1000_resume routine.
5289  */
5290 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
5291 {
5292         struct net_device *netdev = pci_get_drvdata(pdev);
5293         struct e1000_adapter *adapter = netdev_priv(netdev);
5294         struct e1000_hw *hw = &adapter->hw;
5295         int err;
5296
5297         if (adapter->need_ioport)
5298                 err = pci_enable_device(pdev);
5299         else
5300                 err = pci_enable_device_mem(pdev);
5301         if (err) {
5302                 pr_err("Cannot re-enable PCI device after reset.\n");
5303                 return PCI_ERS_RESULT_DISCONNECT;
5304         }
5305         pci_set_master(pdev);
5306
5307         pci_enable_wake(pdev, PCI_D3hot, 0);
5308         pci_enable_wake(pdev, PCI_D3cold, 0);
5309
5310         e1000_reset(adapter);
5311         ew32(WUS, ~0);
5312
5313         return PCI_ERS_RESULT_RECOVERED;
5314 }
5315
5316 /**
5317  * e1000_io_resume - called when traffic can start flowing again.
5318  * @pdev: Pointer to PCI device
5319  *
5320  * This callback is called when the error recovery driver tells us that
5321  * its OK to resume normal operation. Implementation resembles the
5322  * second-half of the e1000_resume routine.
5323  */
5324 static void e1000_io_resume(struct pci_dev *pdev)
5325 {
5326         struct net_device *netdev = pci_get_drvdata(pdev);
5327         struct e1000_adapter *adapter = netdev_priv(netdev);
5328
5329         e1000_init_manageability(adapter);
5330
5331         if (netif_running(netdev)) {
5332                 if (e1000_up(adapter)) {
5333                         pr_info("can't bring device back up after reset\n");
5334                         return;
5335                 }
5336         }
5337
5338         netif_device_attach(netdev);
5339 }
5340
5341 /* e1000_main.c */