drivers/net/ethernet/intel/i40e/i40e_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2014 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 /* Local includes */
  28 #include "i40e.h"
  29 #include "i40e_diag.h"
  30 #ifdef CONFIG_I40E_VXLAN
  31 #include <net/vxlan.h>
  32 #endif
  33
  34 const char i40e_driver_name[] = "i40e";
  35 static const char i40e_driver_string[] =
  36                         "Intel(R) Ethernet Connection XL710 Network Driver";
  37
  38 #define DRV_KERN "-k"
  39
  40 #define DRV_VERSION_MAJOR 0
  41 #define DRV_VERSION_MINOR 4
  42 #define DRV_VERSION_BUILD 3
  43 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  44              __stringify(DRV_VERSION_MINOR) "." \
  45              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  46 const char i40e_driver_version_str[] = DRV_VERSION;
  47 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation.";
  48
  49 /* a bit of forward declarations */
  50 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  51 static void i40e_handle_reset_warning(struct i40e_pf *pf);
  52 static int i40e_add_vsi(struct i40e_vsi *vsi);
  53 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  54 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  55 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  56 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  57 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  58 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  59 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  60
  61 /* i40e_pci_tbl - PCI Device ID Table
  62  *
  63  * Last entry must be all 0s
  64  *
  65  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  66  *   Class, Class Mask, private data (not used) }
  67  */
  68 static DEFINE_PCI_DEVICE_TABLE(i40e_pci_tbl) = {
  69         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  70         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  71         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_A), 0},
  72         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  73         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  74         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  75         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  76         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  77         /* required last entry */
  78         {0, }
  79 };
  80 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  81
  82 #define I40E_MAX_VF_COUNT 128
  83 static int debug = -1;
  84 module_param(debug, int, 0);
  85 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  86
  87 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
  88 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
  89 MODULE_LICENSE("GPL");
  90 MODULE_VERSION(DRV_VERSION);
  91
  92 /**
  93  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
  94  * @hw:   pointer to the HW structure
  95  * @mem:  ptr to mem struct to fill out
  96  * @size: size of memory requested
  97  * @alignment: what to align the allocation to
  98  **/
  99 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 100                             u64 size, u32 alignment)
 101 {
 102         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 103
 104         mem->size = ALIGN(size, alignment);
 105         mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
 106                                       &mem->pa, GFP_KERNEL);
 107         if (!mem->va)
 108                 return -ENOMEM;
 109
 110         return 0;
 111 }
 112
 113 /**
 114  * i40e_free_dma_mem_d - OS specific memory free for shared code
 115  * @hw:   pointer to the HW structure
 116  * @mem:  ptr to mem struct to free
 117  **/
 118 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 119 {
 120         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 121
 122         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 123         mem->va = NULL;
 124         mem->pa = 0;
 125         mem->size = 0;
 126
 127         return 0;
 128 }
 129
 130 /**
 131  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 132  * @hw:   pointer to the HW structure
 133  * @mem:  ptr to mem struct to fill out
 134  * @size: size of memory requested
 135  **/
 136 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 137                              u32 size)
 138 {
 139         mem->size = size;
 140         mem->va = kzalloc(size, GFP_KERNEL);
 141
 142         if (!mem->va)
 143                 return -ENOMEM;
 144
 145         return 0;
 146 }
 147
 148 /**
 149  * i40e_free_virt_mem_d - OS specific memory free for shared code
 150  * @hw:   pointer to the HW structure
 151  * @mem:  ptr to mem struct to free
 152  **/
 153 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 154 {
 155         /* it's ok to kfree a NULL pointer */
 156         kfree(mem->va);
 157         mem->va = NULL;
 158         mem->size = 0;
 159
 160         return 0;
 161 }
 162
 163 /**
 164  * i40e_get_lump - find a lump of free generic resource
 165  * @pf: board private structure
 166  * @pile: the pile of resource to search
 167  * @needed: the number of items needed
 168  * @id: an owner id to stick on the items assigned
 169  *
 170  * Returns the base item index of the lump, or negative for error
 171  *
 172  * The search_hint trick and lack of advanced fit-finding only work
 173  * because we're highly likely to have all the same size lump requests.
 174  * Linear search time and any fragmentation should be minimal.
 175  **/
 176 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 177                          u16 needed, u16 id)
 178 {
 179         int ret = -ENOMEM;
 180         int i, j;
 181
 182         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 183                 dev_info(&pf->pdev->dev,
 184                          "param err: pile=%p needed=%d id=0x%04x\n",
 185                          pile, needed, id);
 186                 return -EINVAL;
 187         }
 188
 189         /* start the linear search with an imperfect hint */
 190         i = pile->search_hint;
 191         while (i < pile->num_entries) {
 192                 /* skip already allocated entries */
 193                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 194                         i++;
 195                         continue;
 196                 }
 197
 198                 /* do we have enough in this lump? */
 199                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 200                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 201                                 break;
 202                 }
 203
 204                 if (j == needed) {
 205                         /* there was enough, so assign it to the requestor */
 206                         for (j = 0; j < needed; j++)
 207                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 208                         ret = i;
 209                         pile->search_hint = i + j;
 210                         break;
 211                 } else {
 212                         /* not enough, so skip over it and continue looking */
 213                         i += j;
 214                 }
 215         }
 216
 217         return ret;
 218 }
 219
 220 /**
 221  * i40e_put_lump - return a lump of generic resource
 222  * @pile: the pile of resource to search
 223  * @index: the base item index
 224  * @id: the owner id of the items assigned
 225  *
 226  * Returns the count of items in the lump
 227  **/
 228 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 229 {
 230         int valid_id = (id | I40E_PILE_VALID_BIT);
 231         int count = 0;
 232         int i;
 233
 234         if (!pile || index >= pile->num_entries)
 235                 return -EINVAL;
 236
 237         for (i = index;
 238              i < pile->num_entries && pile->list[i] == valid_id;
 239              i++) {
 240                 pile->list[i] = 0;
 241                 count++;
 242         }
 243
 244         if (count && index < pile->search_hint)
 245                 pile->search_hint = index;
 246
 247         return count;
 248 }
 249
 250 /**
 251  * i40e_service_event_schedule - Schedule the service task to wake up
 252  * @pf: board private structure
 253  *
 254  * If not already scheduled, this puts the task into the work queue
 255  **/
 256 static void i40e_service_event_schedule(struct i40e_pf *pf)
 257 {
 258         if (!test_bit(__I40E_DOWN, &pf->state) &&
 259             !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state) &&
 260             !test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state))
 261                 schedule_work(&pf->service_task);
 262 }
 263
 264 /**
 265  * i40e_tx_timeout - Respond to a Tx Hang
 266  * @netdev: network interface device structure
 267  *
 268  * If any port has noticed a Tx timeout, it is likely that the whole
 269  * device is munged, not just the one netdev port, so go for the full
 270  * reset.
 271  **/
 272 static void i40e_tx_timeout(struct net_device *netdev)
 273 {
 274         struct i40e_netdev_priv *np = netdev_priv(netdev);
 275         struct i40e_vsi *vsi = np->vsi;
 276         struct i40e_pf *pf = vsi->back;
 277
 278         pf->tx_timeout_count++;
 279
 280         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 281                 pf->tx_timeout_recovery_level = 0;
 282         pf->tx_timeout_last_recovery = jiffies;
 283         netdev_info(netdev, "tx_timeout recovery level %d\n",
 284                     pf->tx_timeout_recovery_level);
 285
 286         switch (pf->tx_timeout_recovery_level) {
 287         case 0:
 288                 /* disable and re-enable queues for the VSI */
 289                 if (in_interrupt()) {
 290                         set_bit(__I40E_REINIT_REQUESTED, &pf->state);
 291                         set_bit(__I40E_REINIT_REQUESTED, &vsi->state);
 292                 } else {
 293                         i40e_vsi_reinit_locked(vsi);
 294                 }
 295                 break;
 296         case 1:
 297                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
 298                 break;
 299         case 2:
 300                 set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
 301                 break;
 302         case 3:
 303                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
 304                 break;
 305         default:
 306                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 307                 set_bit(__I40E_DOWN, &vsi->state);
 308                 i40e_down(vsi);
 309                 break;
 310         }
 311         i40e_service_event_schedule(pf);
 312         pf->tx_timeout_recovery_level++;
 313 }
 314
 315 /**
 316  * i40e_release_rx_desc - Store the new tail and head values
 317  * @rx_ring: ring to bump
 318  * @val: new head index
 319  **/
 320 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
 321 {
 322         rx_ring->next_to_use = val;
 323
 324         /* Force memory writes to complete before letting h/w
 325          * know there are new descriptors to fetch.  (Only
 326          * applicable for weak-ordered memory model archs,
 327          * such as IA-64).
 328          */
 329         wmb();
 330         writel(val, rx_ring->tail);
 331 }
 332
 333 /**
 334  * i40e_get_vsi_stats_struct - Get System Network Statistics
 335  * @vsi: the VSI we care about
 336  *
 337  * Returns the address of the device statistics structure.
 338  * The statistics are actually updated from the service task.
 339  **/
 340 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 341 {
 342         return &vsi->net_stats;
 343 }
 344
 345 /**
 346  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 347  * @netdev: network interface device structure
 348  *
 349  * Returns the address of the device statistics structure.
 350  * The statistics are actually updated from the service task.
 351  **/
 352 static struct rtnl_link_stats64 *i40e_get_netdev_stats_struct(
 353                                              struct net_device *netdev,
 354                                              struct rtnl_link_stats64 *stats)
 355 {
 356         struct i40e_netdev_priv *np = netdev_priv(netdev);
 357         struct i40e_ring *tx_ring, *rx_ring;
 358         struct i40e_vsi *vsi = np->vsi;
 359         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 360         int i;
 361
 362         if (test_bit(__I40E_DOWN, &vsi->state))
 363                 return stats;
 364
 365         if (!vsi->tx_rings)
 366                 return stats;
 367
 368         rcu_read_lock();
 369         for (i = 0; i < vsi->num_queue_pairs; i++) {
 370                 u64 bytes, packets;
 371                 unsigned int start;
 372
 373                 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]);
 374                 if (!tx_ring)
 375                         continue;
 376
 377                 do {
 378                         start = u64_stats_fetch_begin_irq(&tx_ring->syncp);
 379                         packets = tx_ring->stats.packets;
 380                         bytes   = tx_ring->stats.bytes;
 381                 } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start));
 382
 383                 stats->tx_packets += packets;
 384                 stats->tx_bytes   += bytes;
 385                 rx_ring = &tx_ring[1];
 386
 387                 do {
 388                         start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
 389                         packets = rx_ring->stats.packets;
 390                         bytes   = rx_ring->stats.bytes;
 391                 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
 392
 393                 stats->rx_packets += packets;
 394                 stats->rx_bytes   += bytes;
 395         }
 396         rcu_read_unlock();
 397
 398         /* following stats updated by i40e_watchdog_subtask() */
 399         stats->multicast        = vsi_stats->multicast;
 400         stats->tx_errors        = vsi_stats->tx_errors;
 401         stats->tx_dropped       = vsi_stats->tx_dropped;
 402         stats->rx_errors        = vsi_stats->rx_errors;
 403         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 404         stats->rx_length_errors = vsi_stats->rx_length_errors;
 405
 406         return stats;
 407 }
 408
 409 /**
 410  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 411  * @vsi: the VSI to have its stats reset
 412  **/
 413 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 414 {
 415         struct rtnl_link_stats64 *ns;
 416         int i;
 417
 418         if (!vsi)
 419                 return;
 420
 421         ns = i40e_get_vsi_stats_struct(vsi);
 422         memset(ns, 0, sizeof(*ns));
 423         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 424         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 425         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 426         if (vsi->rx_rings && vsi->rx_rings[0]) {
 427                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 428                         memset(&vsi->rx_rings[i]->stats, 0 ,
 429                                sizeof(vsi->rx_rings[i]->stats));
 430                         memset(&vsi->rx_rings[i]->rx_stats, 0 ,
 431                                sizeof(vsi->rx_rings[i]->rx_stats));
 432                         memset(&vsi->tx_rings[i]->stats, 0 ,
 433                                sizeof(vsi->tx_rings[i]->stats));
 434                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 435                                sizeof(vsi->tx_rings[i]->tx_stats));
 436                 }
 437         }
 438         vsi->stat_offsets_loaded = false;
 439 }
 440
 441 /**
 442  * i40e_pf_reset_stats - Reset all of the stats for the given pf
 443  * @pf: the PF to be reset
 444  **/
 445 void i40e_pf_reset_stats(struct i40e_pf *pf)
 446 {
 447         memset(&pf->stats, 0, sizeof(pf->stats));
 448         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 449         pf->stat_offsets_loaded = false;
 450 }
 451
 452 /**
 453  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 454  * @hw: ptr to the hardware info
 455  * @hireg: the high 32 bit reg to read
 456  * @loreg: the low 32 bit reg to read
 457  * @offset_loaded: has the initial offset been loaded yet
 458  * @offset: ptr to current offset value
 459  * @stat: ptr to the stat
 460  *
 461  * Since the device stats are not reset at PFReset, they likely will not
 462  * be zeroed when the driver starts.  We'll save the first values read
 463  * and use them as offsets to be subtracted from the raw values in order
 464  * to report stats that count from zero.  In the process, we also manage
 465  * the potential roll-over.
 466  **/
 467 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 468                                bool offset_loaded, u64 *offset, u64 *stat)
 469 {
 470         u64 new_data;
 471
 472         if (hw->device_id == I40E_DEV_ID_QEMU) {
 473                 new_data = rd32(hw, loreg);
 474                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 475         } else {
 476                 new_data = rd64(hw, loreg);
 477         }
 478         if (!offset_loaded)
 479                 *offset = new_data;
 480         if (likely(new_data >= *offset))
 481                 *stat = new_data - *offset;
 482         else
 483                 *stat = (new_data + ((u64)1 << 48)) - *offset;
 484         *stat &= 0xFFFFFFFFFFFFULL;
 485 }
 486
 487 /**
 488  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 489  * @hw: ptr to the hardware info
 490  * @reg: the hw reg to read
 491  * @offset_loaded: has the initial offset been loaded yet
 492  * @offset: ptr to current offset value
 493  * @stat: ptr to the stat
 494  **/
 495 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 496                                bool offset_loaded, u64 *offset, u64 *stat)
 497 {
 498         u32 new_data;
 499
 500         new_data = rd32(hw, reg);
 501         if (!offset_loaded)
 502                 *offset = new_data;
 503         if (likely(new_data >= *offset))
 504                 *stat = (u32)(new_data - *offset);
 505         else
 506                 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
 507 }
 508
 509 /**
 510  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 511  * @vsi: the VSI to be updated
 512  **/
 513 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 514 {
 515         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 516         struct i40e_pf *pf = vsi->back;
 517         struct i40e_hw *hw = &pf->hw;
 518         struct i40e_eth_stats *oes;
 519         struct i40e_eth_stats *es;     /* device's eth stats */
 520
 521         es = &vsi->eth_stats;
 522         oes = &vsi->eth_stats_offsets;
 523
 524         /* Gather up the stats that the hw collects */
 525         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 526                            vsi->stat_offsets_loaded,
 527                            &oes->tx_errors, &es->tx_errors);
 528         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 529                            vsi->stat_offsets_loaded,
 530                            &oes->rx_discards, &es->rx_discards);
 531         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 532                            vsi->stat_offsets_loaded,
 533                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 534         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 535                            vsi->stat_offsets_loaded,
 536                            &oes->tx_errors, &es->tx_errors);
 537
 538         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 539                            I40E_GLV_GORCL(stat_idx),
 540                            vsi->stat_offsets_loaded,
 541                            &oes->rx_bytes, &es->rx_bytes);
 542         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 543                            I40E_GLV_UPRCL(stat_idx),
 544                            vsi->stat_offsets_loaded,
 545                            &oes->rx_unicast, &es->rx_unicast);
 546         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 547                            I40E_GLV_MPRCL(stat_idx),
 548                            vsi->stat_offsets_loaded,
 549                            &oes->rx_multicast, &es->rx_multicast);
 550         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 551                            I40E_GLV_BPRCL(stat_idx),
 552                            vsi->stat_offsets_loaded,
 553                            &oes->rx_broadcast, &es->rx_broadcast);
 554
 555         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 556                            I40E_GLV_GOTCL(stat_idx),
 557                            vsi->stat_offsets_loaded,
 558                            &oes->tx_bytes, &es->tx_bytes);
 559         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 560                            I40E_GLV_UPTCL(stat_idx),
 561                            vsi->stat_offsets_loaded,
 562                            &oes->tx_unicast, &es->tx_unicast);
 563         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 564                            I40E_GLV_MPTCL(stat_idx),
 565                            vsi->stat_offsets_loaded,
 566                            &oes->tx_multicast, &es->tx_multicast);
 567         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 568                            I40E_GLV_BPTCL(stat_idx),
 569                            vsi->stat_offsets_loaded,
 570                            &oes->tx_broadcast, &es->tx_broadcast);
 571         vsi->stat_offsets_loaded = true;
 572 }
 573
 574 /**
 575  * i40e_update_veb_stats - Update Switch component statistics
 576  * @veb: the VEB being updated
 577  **/
 578 static void i40e_update_veb_stats(struct i40e_veb *veb)
 579 {
 580         struct i40e_pf *pf = veb->pf;
 581         struct i40e_hw *hw = &pf->hw;
 582         struct i40e_eth_stats *oes;
 583         struct i40e_eth_stats *es;     /* device's eth stats */
 584         int idx = 0;
 585
 586         idx = veb->stats_idx;
 587         es = &veb->stats;
 588         oes = &veb->stats_offsets;
 589
 590         /* Gather up the stats that the hw collects */
 591         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 592                            veb->stat_offsets_loaded,
 593                            &oes->tx_discards, &es->tx_discards);
 594         if (hw->revision_id > 0)
 595                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 596                                    veb->stat_offsets_loaded,
 597                                    &oes->rx_unknown_protocol,
 598                                    &es->rx_unknown_protocol);
 599         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 600                            veb->stat_offsets_loaded,
 601                            &oes->rx_bytes, &es->rx_bytes);
 602         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 603                            veb->stat_offsets_loaded,
 604                            &oes->rx_unicast, &es->rx_unicast);
 605         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 606                            veb->stat_offsets_loaded,
 607                            &oes->rx_multicast, &es->rx_multicast);
 608         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 609                            veb->stat_offsets_loaded,
 610                            &oes->rx_broadcast, &es->rx_broadcast);
 611
 612         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 613                            veb->stat_offsets_loaded,
 614                            &oes->tx_bytes, &es->tx_bytes);
 615         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 616                            veb->stat_offsets_loaded,
 617                            &oes->tx_unicast, &es->tx_unicast);
 618         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 619                            veb->stat_offsets_loaded,
 620                            &oes->tx_multicast, &es->tx_multicast);
 621         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 622                            veb->stat_offsets_loaded,
 623                            &oes->tx_broadcast, &es->tx_broadcast);
 624         veb->stat_offsets_loaded = true;
 625 }
 626
 627 /**
 628  * i40e_update_link_xoff_rx - Update XOFF received in link flow control mode
 629  * @pf: the corresponding PF
 630  *
 631  * Update the Rx XOFF counter (PAUSE frames) in link flow control mode
 632  **/
 633 static void i40e_update_link_xoff_rx(struct i40e_pf *pf)
 634 {
 635         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 636         struct i40e_hw_port_stats *nsd = &pf->stats;
 637         struct i40e_hw *hw = &pf->hw;
 638         u64 xoff = 0;
 639         u16 i, v;
 640
 641         if ((hw->fc.current_mode != I40E_FC_FULL) &&
 642             (hw->fc.current_mode != I40E_FC_RX_PAUSE))
 643                 return;
 644
 645         xoff = nsd->link_xoff_rx;
 646         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 647                            pf->stat_offsets_loaded,
 648                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 649
 650         /* No new LFC xoff rx */
 651         if (!(nsd->link_xoff_rx - xoff))
 652                 return;
 653
 654         /* Clear the __I40E_HANG_CHECK_ARMED bit for all Tx rings */
 655         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
 656                 struct i40e_vsi *vsi = pf->vsi[v];
 657
 658                 if (!vsi || !vsi->tx_rings[0])
 659                         continue;
 660
 661                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 662                         struct i40e_ring *ring = vsi->tx_rings[i];
 663                         clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);
 664                 }
 665         }
 666 }
 667
 668 /**
 669  * i40e_update_prio_xoff_rx - Update XOFF received in PFC mode
 670  * @pf: the corresponding PF
 671  *
 672  * Update the Rx XOFF counter (PAUSE frames) in PFC mode
 673  **/
 674 static void i40e_update_prio_xoff_rx(struct i40e_pf *pf)
 675 {
 676         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 677         struct i40e_hw_port_stats *nsd = &pf->stats;
 678         bool xoff[I40E_MAX_TRAFFIC_CLASS] = {false};
 679         struct i40e_dcbx_config *dcb_cfg;
 680         struct i40e_hw *hw = &pf->hw;
 681         u16 i, v;
 682         u8 tc;
 683
 684         dcb_cfg = &hw->local_dcbx_config;
 685
 686         /* See if DCB enabled with PFC TC */
 687         if (!(pf->flags & I40E_FLAG_DCB_ENABLED) ||
 688             !(dcb_cfg->pfc.pfcenable)) {
 689                 i40e_update_link_xoff_rx(pf);
 690                 return;
 691         }
 692
 693         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
 694                 u64 prio_xoff = nsd->priority_xoff_rx[i];
 695                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 696                                    pf->stat_offsets_loaded,
 697                                    &osd->priority_xoff_rx[i],
 698                                    &nsd->priority_xoff_rx[i]);
 699
 700                 /* No new PFC xoff rx */
 701                 if (!(nsd->priority_xoff_rx[i] - prio_xoff))
 702                         continue;
 703                 /* Get the TC for given priority */
 704                 tc = dcb_cfg->etscfg.prioritytable[i];
 705                 xoff[tc] = true;
 706         }
 707
 708         /* Clear the __I40E_HANG_CHECK_ARMED bit for Tx rings */
 709         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
 710                 struct i40e_vsi *vsi = pf->vsi[v];
 711
 712                 if (!vsi || !vsi->tx_rings[0])
 713                         continue;
 714
 715                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 716                         struct i40e_ring *ring = vsi->tx_rings[i];
 717
 718                         tc = ring->dcb_tc;
 719                         if (xoff[tc])
 720                                 clear_bit(__I40E_HANG_CHECK_ARMED,
 721                                           &ring->state);
 722                 }
 723         }
 724 }
 725
 726 /**
 727  * i40e_update_vsi_stats - Update the vsi statistics counters.
 728  * @vsi: the VSI to be updated
 729  *
 730  * There are a few instances where we store the same stat in a
 731  * couple of different structs.  This is partly because we have
 732  * the netdev stats that need to be filled out, which is slightly
 733  * different from the "eth_stats" defined by the chip and used in
 734  * VF communications.  We sort it out here.
 735  **/
 736 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 737 {
 738         struct i40e_pf *pf = vsi->back;
 739         struct rtnl_link_stats64 *ons;
 740         struct rtnl_link_stats64 *ns;   /* netdev stats */
 741         struct i40e_eth_stats *oes;
 742         struct i40e_eth_stats *es;     /* device's eth stats */
 743         u32 tx_restart, tx_busy;
 744         u32 rx_page, rx_buf;
 745         u64 rx_p, rx_b;
 746         u64 tx_p, tx_b;
 747         u16 q;
 748
 749         if (test_bit(__I40E_DOWN, &vsi->state) ||
 750             test_bit(__I40E_CONFIG_BUSY, &pf->state))
 751                 return;
 752
 753         ns = i40e_get_vsi_stats_struct(vsi);
 754         ons = &vsi->net_stats_offsets;
 755         es = &vsi->eth_stats;
 756         oes = &vsi->eth_stats_offsets;
 757
 758         /* Gather up the netdev and vsi stats that the driver collects
 759          * on the fly during packet processing
 760          */
 761         rx_b = rx_p = 0;
 762         tx_b = tx_p = 0;
 763         tx_restart = tx_busy = 0;
 764         rx_page = 0;
 765         rx_buf = 0;
 766         rcu_read_lock();
 767         for (q = 0; q < vsi->num_queue_pairs; q++) {
 768                 struct i40e_ring *p;
 769                 u64 bytes, packets;
 770                 unsigned int start;
 771
 772                 /* locate Tx ring */
 773                 p = ACCESS_ONCE(vsi->tx_rings[q]);
 774
 775                 do {
 776                         start = u64_stats_fetch_begin_irq(&p->syncp);
 777                         packets = p->stats.packets;
 778                         bytes = p->stats.bytes;
 779                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 780                 tx_b += bytes;
 781                 tx_p += packets;
 782                 tx_restart += p->tx_stats.restart_queue;
 783                 tx_busy += p->tx_stats.tx_busy;
 784
 785                 /* Rx queue is part of the same block as Tx queue */
 786                 p = &p[1];
 787                 do {
 788                         start = u64_stats_fetch_begin_irq(&p->syncp);
 789                         packets = p->stats.packets;
 790                         bytes = p->stats.bytes;
 791                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 792                 rx_b += bytes;
 793                 rx_p += packets;
 794                 rx_buf += p->rx_stats.alloc_buff_failed;
 795                 rx_page += p->rx_stats.alloc_page_failed;
 796         }
 797         rcu_read_unlock();
 798         vsi->tx_restart = tx_restart;
 799         vsi->tx_busy = tx_busy;
 800         vsi->rx_page_failed = rx_page;
 801         vsi->rx_buf_failed = rx_buf;
 802
 803         ns->rx_packets = rx_p;
 804         ns->rx_bytes = rx_b;
 805         ns->tx_packets = tx_p;
 806         ns->tx_bytes = tx_b;
 807
 808         /* update netdev stats from eth stats */
 809         i40e_update_eth_stats(vsi);
 810         ons->tx_errors = oes->tx_errors;
 811         ns->tx_errors = es->tx_errors;
 812         ons->multicast = oes->rx_multicast;
 813         ns->multicast = es->rx_multicast;
 814         ons->rx_dropped = oes->rx_discards;
 815         ns->rx_dropped = es->rx_discards;
 816         ons->tx_dropped = oes->tx_discards;
 817         ns->tx_dropped = es->tx_discards;
 818
 819         /* pull in a couple PF stats if this is the main vsi */
 820         if (vsi == pf->vsi[pf->lan_vsi]) {
 821                 ns->rx_crc_errors = pf->stats.crc_errors;
 822                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 823                 ns->rx_length_errors = pf->stats.rx_length_errors;
 824         }
 825 }
 826
 827 /**
 828  * i40e_update_pf_stats - Update the pf statistics counters.
 829  * @pf: the PF to be updated
 830  **/
 831 static void i40e_update_pf_stats(struct i40e_pf *pf)
 832 {
 833         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 834         struct i40e_hw_port_stats *nsd = &pf->stats;
 835         struct i40e_hw *hw = &pf->hw;
 836         u32 val;
 837         int i;
 838
 839         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 840                            I40E_GLPRT_GORCL(hw->port),
 841                            pf->stat_offsets_loaded,
 842                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 843         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 844                            I40E_GLPRT_GOTCL(hw->port),
 845                            pf->stat_offsets_loaded,
 846                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 847         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 848                            pf->stat_offsets_loaded,
 849                            &osd->eth.rx_discards,
 850                            &nsd->eth.rx_discards);
 851         i40e_stat_update32(hw, I40E_GLPRT_TDPC(hw->port),
 852                            pf->stat_offsets_loaded,
 853                            &osd->eth.tx_discards,
 854                            &nsd->eth.tx_discards);
 855
 856         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 857                            I40E_GLPRT_UPRCL(hw->port),
 858                            pf->stat_offsets_loaded,
 859                            &osd->eth.rx_unicast,
 860                            &nsd->eth.rx_unicast);
 861         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 862                            I40E_GLPRT_MPRCL(hw->port),
 863                            pf->stat_offsets_loaded,
 864                            &osd->eth.rx_multicast,
 865                            &nsd->eth.rx_multicast);
 866         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 867                            I40E_GLPRT_BPRCL(hw->port),
 868                            pf->stat_offsets_loaded,
 869                            &osd->eth.rx_broadcast,
 870                            &nsd->eth.rx_broadcast);
 871         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 872                            I40E_GLPRT_UPTCL(hw->port),
 873                            pf->stat_offsets_loaded,
 874                            &osd->eth.tx_unicast,
 875                            &nsd->eth.tx_unicast);
 876         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 877                            I40E_GLPRT_MPTCL(hw->port),
 878                            pf->stat_offsets_loaded,
 879                            &osd->eth.tx_multicast,
 880                            &nsd->eth.tx_multicast);
 881         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 882                            I40E_GLPRT_BPTCL(hw->port),
 883                            pf->stat_offsets_loaded,
 884                            &osd->eth.tx_broadcast,
 885                            &nsd->eth.tx_broadcast);
 886
 887         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 888                            pf->stat_offsets_loaded,
 889                            &osd->tx_dropped_link_down,
 890                            &nsd->tx_dropped_link_down);
 891
 892         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 893                            pf->stat_offsets_loaded,
 894                            &osd->crc_errors, &nsd->crc_errors);
 895
 896         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 897                            pf->stat_offsets_loaded,
 898                            &osd->illegal_bytes, &nsd->illegal_bytes);
 899
 900         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 901                            pf->stat_offsets_loaded,
 902                            &osd->mac_local_faults,
 903                            &nsd->mac_local_faults);
 904         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 905                            pf->stat_offsets_loaded,
 906                            &osd->mac_remote_faults,
 907                            &nsd->mac_remote_faults);
 908
 909         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 910                            pf->stat_offsets_loaded,
 911                            &osd->rx_length_errors,
 912                            &nsd->rx_length_errors);
 913
 914         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 915                            pf->stat_offsets_loaded,
 916                            &osd->link_xon_rx, &nsd->link_xon_rx);
 917         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 918                            pf->stat_offsets_loaded,
 919                            &osd->link_xon_tx, &nsd->link_xon_tx);
 920         i40e_update_prio_xoff_rx(pf);  /* handles I40E_GLPRT_LXOFFRXC */
 921         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 922                            pf->stat_offsets_loaded,
 923                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
 924
 925         for (i = 0; i < 8; i++) {
 926                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 927                                    pf->stat_offsets_loaded,
 928                                    &osd->priority_xon_rx[i],
 929                                    &nsd->priority_xon_rx[i]);
 930                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 931                                    pf->stat_offsets_loaded,
 932                                    &osd->priority_xon_tx[i],
 933                                    &nsd->priority_xon_tx[i]);
 934                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 935                                    pf->stat_offsets_loaded,
 936                                    &osd->priority_xoff_tx[i],
 937                                    &nsd->priority_xoff_tx[i]);
 938                 i40e_stat_update32(hw,
 939                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
 940                                    pf->stat_offsets_loaded,
 941                                    &osd->priority_xon_2_xoff[i],
 942                                    &nsd->priority_xon_2_xoff[i]);
 943         }
 944
 945         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
 946                            I40E_GLPRT_PRC64L(hw->port),
 947                            pf->stat_offsets_loaded,
 948                            &osd->rx_size_64, &nsd->rx_size_64);
 949         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
 950                            I40E_GLPRT_PRC127L(hw->port),
 951                            pf->stat_offsets_loaded,
 952                            &osd->rx_size_127, &nsd->rx_size_127);
 953         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
 954                            I40E_GLPRT_PRC255L(hw->port),
 955                            pf->stat_offsets_loaded,
 956                            &osd->rx_size_255, &nsd->rx_size_255);
 957         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
 958                            I40E_GLPRT_PRC511L(hw->port),
 959                            pf->stat_offsets_loaded,
 960                            &osd->rx_size_511, &nsd->rx_size_511);
 961         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
 962                            I40E_GLPRT_PRC1023L(hw->port),
 963                            pf->stat_offsets_loaded,
 964                            &osd->rx_size_1023, &nsd->rx_size_1023);
 965         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
 966                            I40E_GLPRT_PRC1522L(hw->port),
 967                            pf->stat_offsets_loaded,
 968                            &osd->rx_size_1522, &nsd->rx_size_1522);
 969         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
 970                            I40E_GLPRT_PRC9522L(hw->port),
 971                            pf->stat_offsets_loaded,
 972                            &osd->rx_size_big, &nsd->rx_size_big);
 973
 974         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
 975                            I40E_GLPRT_PTC64L(hw->port),
 976                            pf->stat_offsets_loaded,
 977                            &osd->tx_size_64, &nsd->tx_size_64);
 978         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
 979                            I40E_GLPRT_PTC127L(hw->port),
 980                            pf->stat_offsets_loaded,
 981                            &osd->tx_size_127, &nsd->tx_size_127);
 982         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
 983                            I40E_GLPRT_PTC255L(hw->port),
 984                            pf->stat_offsets_loaded,
 985                            &osd->tx_size_255, &nsd->tx_size_255);
 986         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
 987                            I40E_GLPRT_PTC511L(hw->port),
 988                            pf->stat_offsets_loaded,
 989                            &osd->tx_size_511, &nsd->tx_size_511);
 990         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
 991                            I40E_GLPRT_PTC1023L(hw->port),
 992                            pf->stat_offsets_loaded,
 993                            &osd->tx_size_1023, &nsd->tx_size_1023);
 994         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
 995                            I40E_GLPRT_PTC1522L(hw->port),
 996                            pf->stat_offsets_loaded,
 997                            &osd->tx_size_1522, &nsd->tx_size_1522);
 998         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
 999                            I40E_GLPRT_PTC9522L(hw->port),
1000                            pf->stat_offsets_loaded,
1001                            &osd->tx_size_big, &nsd->tx_size_big);
1002
1003         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1004                            pf->stat_offsets_loaded,
1005                            &osd->rx_undersize, &nsd->rx_undersize);
1006         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1007                            pf->stat_offsets_loaded,
1008                            &osd->rx_fragments, &nsd->rx_fragments);
1009         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1010                            pf->stat_offsets_loaded,
1011                            &osd->rx_oversize, &nsd->rx_oversize);
1012         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1013                            pf->stat_offsets_loaded,
1014                            &osd->rx_jabber, &nsd->rx_jabber);
1015
1016         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1017         nsd->tx_lpi_status =
1018                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1019                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1020         nsd->rx_lpi_status =
1021                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1022                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1023         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1024                            pf->stat_offsets_loaded,
1025                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1026         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1027                            pf->stat_offsets_loaded,
1028                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1029
1030         pf->stat_offsets_loaded = true;
1031 }
1032
1033 /**
1034  * i40e_update_stats - Update the various statistics counters.
1035  * @vsi: the VSI to be updated
1036  *
1037  * Update the various stats for this VSI and its related entities.
1038  **/
1039 void i40e_update_stats(struct i40e_vsi *vsi)
1040 {
1041         struct i40e_pf *pf = vsi->back;
1042
1043         if (vsi == pf->vsi[pf->lan_vsi])
1044                 i40e_update_pf_stats(pf);
1045
1046         i40e_update_vsi_stats(vsi);
1047 }
1048
1049 /**
1050  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1051  * @vsi: the VSI to be searched
1052  * @macaddr: the MAC address
1053  * @vlan: the vlan
1054  * @is_vf: make sure its a vf filter, else doesn't matter
1055  * @is_netdev: make sure its a netdev filter, else doesn't matter
1056  *
1057  * Returns ptr to the filter object or NULL
1058  **/
1059 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1060                                                 u8 *macaddr, s16 vlan,
1061                                                 bool is_vf, bool is_netdev)
1062 {
1063         struct i40e_mac_filter *f;
1064
1065         if (!vsi || !macaddr)
1066                 return NULL;
1067
1068         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1069                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1070                     (vlan == f->vlan)    &&
1071                     (!is_vf || f->is_vf) &&
1072                     (!is_netdev || f->is_netdev))
1073                         return f;
1074         }
1075         return NULL;
1076 }
1077
1078 /**
1079  * i40e_find_mac - Find a mac addr in the macvlan filters list
1080  * @vsi: the VSI to be searched
1081  * @macaddr: the MAC address we are searching for
1082  * @is_vf: make sure its a vf filter, else doesn't matter
1083  * @is_netdev: make sure its a netdev filter, else doesn't matter
1084  *
1085  * Returns the first filter with the provided MAC address or NULL if
1086  * MAC address was not found
1087  **/
1088 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, u8 *macaddr,
1089                                       bool is_vf, bool is_netdev)
1090 {
1091         struct i40e_mac_filter *f;
1092
1093         if (!vsi || !macaddr)
1094                 return NULL;
1095
1096         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1097                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1098                     (!is_vf || f->is_vf) &&
1099                     (!is_netdev || f->is_netdev))
1100                         return f;
1101         }
1102         return NULL;
1103 }
1104
1105 /**
1106  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1107  * @vsi: the VSI to be searched
1108  *
1109  * Returns true if VSI is in vlan mode or false otherwise
1110  **/
1111 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1112 {
1113         struct i40e_mac_filter *f;
1114
1115         /* Only -1 for all the filters denotes not in vlan mode
1116          * so we have to go through all the list in order to make sure
1117          */
1118         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1119                 if (f->vlan >= 0)
1120                         return true;
1121         }
1122
1123         return false;
1124 }
1125
1126 /**
1127  * i40e_put_mac_in_vlan - Make macvlan filters from macaddrs and vlans
1128  * @vsi: the VSI to be searched
1129  * @macaddr: the mac address to be filtered
1130  * @is_vf: true if it is a vf
1131  * @is_netdev: true if it is a netdev
1132  *
1133  * Goes through all the macvlan filters and adds a
1134  * macvlan filter for each unique vlan that already exists
1135  *
1136  * Returns first filter found on success, else NULL
1137  **/
1138 struct i40e_mac_filter *i40e_put_mac_in_vlan(struct i40e_vsi *vsi, u8 *macaddr,
1139                                              bool is_vf, bool is_netdev)
1140 {
1141         struct i40e_mac_filter *f;
1142
1143         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1144                 if (!i40e_find_filter(vsi, macaddr, f->vlan,
1145                                       is_vf, is_netdev)) {
1146                         if (!i40e_add_filter(vsi, macaddr, f->vlan,
1147                                              is_vf, is_netdev))
1148                                 return NULL;
1149                 }
1150         }
1151
1152         return list_first_entry_or_null(&vsi->mac_filter_list,
1153                                         struct i40e_mac_filter, list);
1154 }
1155
1156 /**
1157  * i40e_add_filter - Add a mac/vlan filter to the VSI
1158  * @vsi: the VSI to be searched
1159  * @macaddr: the MAC address
1160  * @vlan: the vlan
1161  * @is_vf: make sure its a vf filter, else doesn't matter
1162  * @is_netdev: make sure its a netdev filter, else doesn't matter
1163  *
1164  * Returns ptr to the filter object or NULL when no memory available.
1165  **/
1166 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1167                                         u8 *macaddr, s16 vlan,
1168                                         bool is_vf, bool is_netdev)
1169 {
1170         struct i40e_mac_filter *f;
1171
1172         if (!vsi || !macaddr)
1173                 return NULL;
1174
1175         f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
1176         if (!f) {
1177                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1178                 if (!f)
1179                         goto add_filter_out;
1180
1181                 memcpy(f->macaddr, macaddr, ETH_ALEN);
1182                 f->vlan = vlan;
1183                 f->changed = true;
1184
1185                 INIT_LIST_HEAD(&f->list);
1186                 list_add(&f->list, &vsi->mac_filter_list);
1187         }
1188
1189         /* increment counter and add a new flag if needed */
1190         if (is_vf) {
1191                 if (!f->is_vf) {
1192                         f->is_vf = true;
1193                         f->counter++;
1194                 }
1195         } else if (is_netdev) {
1196                 if (!f->is_netdev) {
1197                         f->is_netdev = true;
1198                         f->counter++;
1199                 }
1200         } else {
1201                 f->counter++;
1202         }
1203
1204         /* changed tells sync_filters_subtask to
1205          * push the filter down to the firmware
1206          */
1207         if (f->changed) {
1208                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1209                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1210         }
1211
1212 add_filter_out:
1213         return f;
1214 }
1215
1216 /**
1217  * i40e_del_filter - Remove a mac/vlan filter from the VSI
1218  * @vsi: the VSI to be searched
1219  * @macaddr: the MAC address
1220  * @vlan: the vlan
1221  * @is_vf: make sure it's a vf filter, else doesn't matter
1222  * @is_netdev: make sure it's a netdev filter, else doesn't matter
1223  **/
1224 void i40e_del_filter(struct i40e_vsi *vsi,
1225                      u8 *macaddr, s16 vlan,
1226                      bool is_vf, bool is_netdev)
1227 {
1228         struct i40e_mac_filter *f;
1229
1230         if (!vsi || !macaddr)
1231                 return;
1232
1233         f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
1234         if (!f || f->counter == 0)
1235                 return;
1236
1237         if (is_vf) {
1238                 if (f->is_vf) {
1239                         f->is_vf = false;
1240                         f->counter--;
1241                 }
1242         } else if (is_netdev) {
1243                 if (f->is_netdev) {
1244                         f->is_netdev = false;
1245                         f->counter--;
1246                 }
1247         } else {
1248                 /* make sure we don't remove a filter in use by vf or netdev */
1249                 int min_f = 0;
1250                 min_f += (f->is_vf ? 1 : 0);
1251                 min_f += (f->is_netdev ? 1 : 0);
1252
1253                 if (f->counter > min_f)
1254                         f->counter--;
1255         }
1256
1257         /* counter == 0 tells sync_filters_subtask to
1258          * remove the filter from the firmware's list
1259          */
1260         if (f->counter == 0) {
1261                 f->changed = true;
1262                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1263                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1264         }
1265 }
1266
1267 /**
1268  * i40e_set_mac - NDO callback to set mac address
1269  * @netdev: network interface device structure
1270  * @p: pointer to an address structure
1271  *
1272  * Returns 0 on success, negative on failure
1273  **/
1274 static int i40e_set_mac(struct net_device *netdev, void *p)
1275 {
1276         struct i40e_netdev_priv *np = netdev_priv(netdev);
1277         struct i40e_vsi *vsi = np->vsi;
1278         struct sockaddr *addr = p;
1279         struct i40e_mac_filter *f;
1280
1281         if (!is_valid_ether_addr(addr->sa_data))
1282                 return -EADDRNOTAVAIL;
1283
1284         netdev_info(netdev, "set mac address=%pM\n", addr->sa_data);
1285
1286         if (ether_addr_equal(netdev->dev_addr, addr->sa_data))
1287                 return 0;
1288
1289         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1290             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1291                 return -EADDRNOTAVAIL;
1292
1293         if (vsi->type == I40E_VSI_MAIN) {
1294                 i40e_status ret;
1295                 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1296                                                 I40E_AQC_WRITE_TYPE_LAA_ONLY,
1297                                                 addr->sa_data, NULL);
1298                 if (ret) {
1299                         netdev_info(netdev,
1300                                     "Addr change for Main VSI failed: %d\n",
1301                                     ret);
1302                         return -EADDRNOTAVAIL;
1303                 }
1304
1305                 memcpy(vsi->back->hw.mac.addr, addr->sa_data, netdev->addr_len);
1306         }
1307
1308         /* In order to be sure to not drop any packets, add the new address
1309          * then delete the old one.
1310          */
1311         f = i40e_add_filter(vsi, addr->sa_data, I40E_VLAN_ANY, false, false);
1312         if (!f)
1313                 return -ENOMEM;
1314
1315         i40e_sync_vsi_filters(vsi);
1316         i40e_del_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY, false, false);
1317         i40e_sync_vsi_filters(vsi);
1318
1319         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
1320
1321         return 0;
1322 }
1323
1324 /**
1325  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1326  * @vsi: the VSI being setup
1327  * @ctxt: VSI context structure
1328  * @enabled_tc: Enabled TCs bitmap
1329  * @is_add: True if called before Add VSI
1330  *
1331  * Setup VSI queue mapping for enabled traffic classes.
1332  **/
1333 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1334                                      struct i40e_vsi_context *ctxt,
1335                                      u8 enabled_tc,
1336                                      bool is_add)
1337 {
1338         struct i40e_pf *pf = vsi->back;
1339         u16 sections = 0;
1340         u8 netdev_tc = 0;
1341         u16 numtc = 0;
1342         u16 qcount;
1343         u8 offset;
1344         u16 qmap;
1345         int i;
1346         u16 num_tc_qps = 0;
1347
1348         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1349         offset = 0;
1350
1351         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1352                 /* Find numtc from enabled TC bitmap */
1353                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1354                         if (enabled_tc & (1 << i)) /* TC is enabled */
1355                                 numtc++;
1356                 }
1357                 if (!numtc) {
1358                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1359                         numtc = 1;
1360                 }
1361         } else {
1362                 /* At least TC0 is enabled in case of non-DCB case */
1363                 numtc = 1;
1364         }
1365
1366         vsi->tc_config.numtc = numtc;
1367         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1368         /* Number of queues per enabled TC */
1369         num_tc_qps = rounddown_pow_of_two(vsi->alloc_queue_pairs/numtc);
1370         num_tc_qps = min_t(int, num_tc_qps, I40E_MAX_QUEUES_PER_TC);
1371
1372         /* Setup queue offset/count for all TCs for given VSI */
1373         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1374                 /* See if the given TC is enabled for the given VSI */
1375                 if (vsi->tc_config.enabled_tc & (1 << i)) { /* TC is enabled */
1376                         int pow, num_qps;
1377
1378                         switch (vsi->type) {
1379                         case I40E_VSI_MAIN:
1380                                 qcount = min_t(int, pf->rss_size, num_tc_qps);
1381                                 break;
1382                         case I40E_VSI_FDIR:
1383                         case I40E_VSI_SRIOV:
1384                         case I40E_VSI_VMDQ2:
1385                         default:
1386                                 qcount = num_tc_qps;
1387                                 WARN_ON(i != 0);
1388                                 break;
1389                         }
1390                         vsi->tc_config.tc_info[i].qoffset = offset;
1391                         vsi->tc_config.tc_info[i].qcount = qcount;
1392
1393                         /* find the power-of-2 of the number of queue pairs */
1394                         num_qps = qcount;
1395                         pow = 0;
1396                         while (num_qps && ((1 << pow) < qcount)) {
1397                                 pow++;
1398                                 num_qps >>= 1;
1399                         }
1400
1401                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1402                         qmap =
1403                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1404                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1405
1406                         offset += qcount;
1407                 } else {
1408                         /* TC is not enabled so set the offset to
1409                          * default queue and allocate one queue
1410                          * for the given TC.
1411                          */
1412                         vsi->tc_config.tc_info[i].qoffset = 0;
1413                         vsi->tc_config.tc_info[i].qcount = 1;
1414                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1415
1416                         qmap = 0;
1417                 }
1418                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1419         }
1420
1421         /* Set actual Tx/Rx queue pairs */
1422         vsi->num_queue_pairs = offset;
1423
1424         /* Scheduler section valid can only be set for ADD VSI */
1425         if (is_add) {
1426                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1427
1428                 ctxt->info.up_enable_bits = enabled_tc;
1429         }
1430         if (vsi->type == I40E_VSI_SRIOV) {
1431                 ctxt->info.mapping_flags |=
1432                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1433                 for (i = 0; i < vsi->num_queue_pairs; i++)
1434                         ctxt->info.queue_mapping[i] =
1435                                                cpu_to_le16(vsi->base_queue + i);
1436         } else {
1437                 ctxt->info.mapping_flags |=
1438                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1439                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1440         }
1441         ctxt->info.valid_sections |= cpu_to_le16(sections);
1442 }
1443
1444 /**
1445  * i40e_set_rx_mode - NDO callback to set the netdev filters
1446  * @netdev: network interface device structure
1447  **/
1448 static void i40e_set_rx_mode(struct net_device *netdev)
1449 {
1450         struct i40e_netdev_priv *np = netdev_priv(netdev);
1451         struct i40e_mac_filter *f, *ftmp;
1452         struct i40e_vsi *vsi = np->vsi;
1453         struct netdev_hw_addr *uca;
1454         struct netdev_hw_addr *mca;
1455         struct netdev_hw_addr *ha;
1456
1457         /* add addr if not already in the filter list */
1458         netdev_for_each_uc_addr(uca, netdev) {
1459                 if (!i40e_find_mac(vsi, uca->addr, false, true)) {
1460                         if (i40e_is_vsi_in_vlan(vsi))
1461                                 i40e_put_mac_in_vlan(vsi, uca->addr,
1462                                                      false, true);
1463                         else
1464                                 i40e_add_filter(vsi, uca->addr, I40E_VLAN_ANY,
1465                                                 false, true);
1466                 }
1467         }
1468
1469         netdev_for_each_mc_addr(mca, netdev) {
1470                 if (!i40e_find_mac(vsi, mca->addr, false, true)) {
1471                         if (i40e_is_vsi_in_vlan(vsi))
1472                                 i40e_put_mac_in_vlan(vsi, mca->addr,
1473                                                      false, true);
1474                         else
1475                                 i40e_add_filter(vsi, mca->addr, I40E_VLAN_ANY,
1476                                                 false, true);
1477                 }
1478         }
1479
1480         /* remove filter if not in netdev list */
1481         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1482                 bool found = false;
1483
1484                 if (!f->is_netdev)
1485                         continue;
1486
1487                 if (is_multicast_ether_addr(f->macaddr)) {
1488                         netdev_for_each_mc_addr(mca, netdev) {
1489                                 if (ether_addr_equal(mca->addr, f->macaddr)) {
1490                                         found = true;
1491                                         break;
1492                                 }
1493                         }
1494                 } else {
1495                         netdev_for_each_uc_addr(uca, netdev) {
1496                                 if (ether_addr_equal(uca->addr, f->macaddr)) {
1497                                         found = true;
1498                                         break;
1499                                 }
1500                         }
1501
1502                         for_each_dev_addr(netdev, ha) {
1503                                 if (ether_addr_equal(ha->addr, f->macaddr)) {
1504                                         found = true;
1505                                         break;
1506                                 }
1507                         }
1508                 }
1509                 if (!found)
1510                         i40e_del_filter(
1511                            vsi, f->macaddr, I40E_VLAN_ANY, false, true);
1512         }
1513
1514         /* check for other flag changes */
1515         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1516                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1517                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1518         }
1519 }
1520
1521 /**
1522  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
1523  * @vsi: ptr to the VSI
1524  *
1525  * Push any outstanding VSI filter changes through the AdminQ.
1526  *
1527  * Returns 0 or error value
1528  **/
1529 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
1530 {
1531         struct i40e_mac_filter *f, *ftmp;
1532         bool promisc_forced_on = false;
1533         bool add_happened = false;
1534         int filter_list_len = 0;
1535         u32 changed_flags = 0;
1536         i40e_status aq_ret = 0;
1537         struct i40e_pf *pf;
1538         int num_add = 0;
1539         int num_del = 0;
1540         u16 cmd_flags;
1541
1542         /* empty array typed pointers, kcalloc later */
1543         struct i40e_aqc_add_macvlan_element_data *add_list;
1544         struct i40e_aqc_remove_macvlan_element_data *del_list;
1545
1546         while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state))
1547                 usleep_range(1000, 2000);
1548         pf = vsi->back;
1549
1550         if (vsi->netdev) {
1551                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
1552                 vsi->current_netdev_flags = vsi->netdev->flags;
1553         }
1554
1555         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
1556                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
1557
1558                 filter_list_len = pf->hw.aq.asq_buf_size /
1559                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
1560                 del_list = kcalloc(filter_list_len,
1561                             sizeof(struct i40e_aqc_remove_macvlan_element_data),
1562                             GFP_KERNEL);
1563                 if (!del_list)
1564                         return -ENOMEM;
1565
1566                 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1567                         if (!f->changed)
1568                                 continue;
1569
1570                         if (f->counter != 0)
1571                                 continue;
1572                         f->changed = false;
1573                         cmd_flags = 0;
1574
1575                         /* add to delete list */
1576                         memcpy(del_list[num_del].mac_addr,
1577                                f->macaddr, ETH_ALEN);
1578                         del_list[num_del].vlan_tag =
1579                                 cpu_to_le16((u16)(f->vlan ==
1580                                             I40E_VLAN_ANY ? 0 : f->vlan));
1581
1582                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1583                         del_list[num_del].flags = cmd_flags;
1584                         num_del++;
1585
1586                         /* unlink from filter list */
1587                         list_del(&f->list);
1588                         kfree(f);
1589
1590                         /* flush a full buffer */
1591                         if (num_del == filter_list_len) {
1592                                 aq_ret = i40e_aq_remove_macvlan(&pf->hw,
1593                                             vsi->seid, del_list, num_del,
1594                                             NULL);
1595                                 num_del = 0;
1596                                 memset(del_list, 0, sizeof(*del_list));
1597
1598                                 if (aq_ret)
1599                                         dev_info(&pf->pdev->dev,
1600                                                  "ignoring delete macvlan error, err %d, aq_err %d while flushing a full buffer\n",
1601                                                  aq_ret,
1602                                                  pf->hw.aq.asq_last_status);
1603                         }
1604                 }
1605                 if (num_del) {
1606                         aq_ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid,
1607                                                      del_list, num_del, NULL);
1608                         num_del = 0;
1609
1610                         if (aq_ret)
1611                                 dev_info(&pf->pdev->dev,
1612                                          "ignoring delete macvlan error, err %d, aq_err %d\n",
1613                                          aq_ret, pf->hw.aq.asq_last_status);
1614                 }
1615
1616                 kfree(del_list);
1617                 del_list = NULL;
1618
1619                 /* do all the adds now */
1620                 filter_list_len = pf->hw.aq.asq_buf_size /
1621                                sizeof(struct i40e_aqc_add_macvlan_element_data),
1622                 add_list = kcalloc(filter_list_len,
1623                                sizeof(struct i40e_aqc_add_macvlan_element_data),
1624                                GFP_KERNEL);
1625                 if (!add_list)
1626                         return -ENOMEM;
1627
1628                 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1629                         if (!f->changed)
1630                                 continue;
1631
1632                         if (f->counter == 0)
1633                                 continue;
1634                         f->changed = false;
1635                         add_happened = true;
1636                         cmd_flags = 0;
1637
1638                         /* add to add array */
1639                         memcpy(add_list[num_add].mac_addr,
1640                                f->macaddr, ETH_ALEN);
1641                         add_list[num_add].vlan_tag =
1642                                 cpu_to_le16(
1643                                  (u16)(f->vlan == I40E_VLAN_ANY ? 0 : f->vlan));
1644                         add_list[num_add].queue_number = 0;
1645
1646                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
1647                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
1648                         num_add++;
1649
1650                         /* flush a full buffer */
1651                         if (num_add == filter_list_len) {
1652                                 aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
1653                                                              add_list, num_add,
1654                                                              NULL);
1655                                 num_add = 0;
1656
1657                                 if (aq_ret)
1658                                         break;
1659                                 memset(add_list, 0, sizeof(*add_list));
1660                         }
1661                 }
1662                 if (num_add) {
1663                         aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
1664                                                      add_list, num_add, NULL);
1665                         num_add = 0;
1666                 }
1667                 kfree(add_list);
1668                 add_list = NULL;
1669
1670                 if (add_happened && (!aq_ret)) {
1671                         /* do nothing */;
1672                 } else if (add_happened && (aq_ret)) {
1673                         dev_info(&pf->pdev->dev,
1674                                  "add filter failed, err %d, aq_err %d\n",
1675                                  aq_ret, pf->hw.aq.asq_last_status);
1676                         if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOSPC) &&
1677                             !test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1678                                       &vsi->state)) {
1679                                 promisc_forced_on = true;
1680                                 set_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1681                                         &vsi->state);
1682                                 dev_info(&pf->pdev->dev, "promiscuous mode forced on\n");
1683                         }
1684                 }
1685         }
1686
1687         /* check for changes in promiscuous modes */
1688         if (changed_flags & IFF_ALLMULTI) {
1689                 bool cur_multipromisc;
1690                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
1691                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
1692                                                                vsi->seid,
1693                                                                cur_multipromisc,
1694                                                                NULL);
1695                 if (aq_ret)
1696                         dev_info(&pf->pdev->dev,
1697                                  "set multi promisc failed, err %d, aq_err %d\n",
1698                                  aq_ret, pf->hw.aq.asq_last_status);
1699         }
1700         if ((changed_flags & IFF_PROMISC) || promisc_forced_on) {
1701                 bool cur_promisc;
1702                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
1703                                test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1704                                         &vsi->state));
1705                 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(&vsi->back->hw,
1706                                                              vsi->seid,
1707                                                              cur_promisc, NULL);
1708                 if (aq_ret)
1709                         dev_info(&pf->pdev->dev,
1710                                  "set uni promisc failed, err %d, aq_err %d\n",
1711                                  aq_ret, pf->hw.aq.asq_last_status);
1712                 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
1713                                                    vsi->seid,
1714                                                    cur_promisc, NULL);
1715                 if (aq_ret)
1716                         dev_info(&pf->pdev->dev,
1717                                  "set brdcast promisc failed, err %d, aq_err %d\n",
1718                                  aq_ret, pf->hw.aq.asq_last_status);
1719         }
1720
1721         clear_bit(__I40E_CONFIG_BUSY, &vsi->state);
1722         return 0;
1723 }
1724
1725 /**
1726  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
1727  * @pf: board private structure
1728  **/
1729 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
1730 {
1731         int v;
1732
1733         if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
1734                 return;
1735         pf->flags &= ~I40E_FLAG_FILTER_SYNC;
1736
1737         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
1738                 if (pf->vsi[v] &&
1739                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED))
1740                         i40e_sync_vsi_filters(pf->vsi[v]);
1741         }
1742 }
1743
1744 /**
1745  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
1746  * @netdev: network interface device structure
1747  * @new_mtu: new value for maximum frame size
1748  *
1749  * Returns 0 on success, negative on failure
1750  **/
1751 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
1752 {
1753         struct i40e_netdev_priv *np = netdev_priv(netdev);
1754         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
1755         struct i40e_vsi *vsi = np->vsi;
1756
1757         /* MTU < 68 is an error and causes problems on some kernels */
1758         if ((new_mtu < 68) || (max_frame > I40E_MAX_RXBUFFER))
1759                 return -EINVAL;
1760
1761         netdev_info(netdev, "changing MTU from %d to %d\n",
1762                     netdev->mtu, new_mtu);
1763         netdev->mtu = new_mtu;
1764         if (netif_running(netdev))
1765                 i40e_vsi_reinit_locked(vsi);
1766
1767         return 0;
1768 }
1769
1770 /**
1771  * i40e_ioctl - Access the hwtstamp interface
1772  * @netdev: network interface device structure
1773  * @ifr: interface request data
1774  * @cmd: ioctl command
1775  **/
1776 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
1777 {
1778         struct i40e_netdev_priv *np = netdev_priv(netdev);
1779         struct i40e_pf *pf = np->vsi->back;
1780
1781         switch (cmd) {
1782         case SIOCGHWTSTAMP:
1783                 return i40e_ptp_get_ts_config(pf, ifr);
1784         case SIOCSHWTSTAMP:
1785                 return i40e_ptp_set_ts_config(pf, ifr);
1786         default:
1787                 return -EOPNOTSUPP;
1788         }
1789 }
1790
1791 /**
1792  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
1793  * @vsi: the vsi being adjusted
1794  **/
1795 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
1796 {
1797         struct i40e_vsi_context ctxt;
1798         i40e_status ret;
1799
1800         if ((vsi->info.valid_sections &
1801              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
1802             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
1803                 return;  /* already enabled */
1804
1805         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
1806         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
1807                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
1808
1809         ctxt.seid = vsi->seid;
1810         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
1811         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
1812         if (ret) {
1813                 dev_info(&vsi->back->pdev->dev,
1814                          "%s: update vsi failed, aq_err=%d\n",
1815                          __func__, vsi->back->hw.aq.asq_last_status);
1816         }
1817 }
1818
1819 /**
1820  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
1821  * @vsi: the vsi being adjusted
1822  **/
1823 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
1824 {
1825         struct i40e_vsi_context ctxt;
1826         i40e_status ret;
1827
1828         if ((vsi->info.valid_sections &
1829              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
1830             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
1831              I40E_AQ_VSI_PVLAN_EMOD_MASK))
1832                 return;  /* already disabled */
1833
1834         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
1835         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
1836                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
1837
1838         ctxt.seid = vsi->seid;
1839         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
1840         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
1841         if (ret) {
1842                 dev_info(&vsi->back->pdev->dev,
1843                          "%s: update vsi failed, aq_err=%d\n",
1844                          __func__, vsi->back->hw.aq.asq_last_status);
1845         }
1846 }
1847
1848 /**
1849  * i40e_vlan_rx_register - Setup or shutdown vlan offload
1850  * @netdev: network interface to be adjusted
1851  * @features: netdev features to test if VLAN offload is enabled or not
1852  **/
1853 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
1854 {
1855         struct i40e_netdev_priv *np = netdev_priv(netdev);
1856         struct i40e_vsi *vsi = np->vsi;
1857
1858         if (features & NETIF_F_HW_VLAN_CTAG_RX)
1859                 i40e_vlan_stripping_enable(vsi);
1860         else
1861                 i40e_vlan_stripping_disable(vsi);
1862 }
1863
1864 /**
1865  * i40e_vsi_add_vlan - Add vsi membership for given vlan
1866  * @vsi: the vsi being configured
1867  * @vid: vlan id to be added (0 = untagged only , -1 = any)
1868  **/
1869 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, s16 vid)
1870 {
1871         struct i40e_mac_filter *f, *add_f;
1872         bool is_netdev, is_vf;
1873
1874         is_vf = (vsi->type == I40E_VSI_SRIOV);
1875         is_netdev = !!(vsi->netdev);
1876
1877         if (is_netdev) {
1878                 add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, vid,
1879                                         is_vf, is_netdev);
1880                 if (!add_f) {
1881                         dev_info(&vsi->back->pdev->dev,
1882                                  "Could not add vlan filter %d for %pM\n",
1883                                  vid, vsi->netdev->dev_addr);
1884                         return -ENOMEM;
1885                 }
1886         }
1887
1888         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1889                 add_f = i40e_add_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
1890                 if (!add_f) {
1891                         dev_info(&vsi->back->pdev->dev,
1892                                  "Could not add vlan filter %d for %pM\n",
1893                                  vid, f->macaddr);
1894                         return -ENOMEM;
1895                 }
1896         }
1897
1898         /* Now if we add a vlan tag, make sure to check if it is the first
1899          * tag (i.e. a "tag" -1 does exist) and if so replace the -1 "tag"
1900          * with 0, so we now accept untagged and specified tagged traffic
1901          * (and not any taged and untagged)
1902          */
1903         if (vid > 0) {
1904                 if (is_netdev && i40e_find_filter(vsi, vsi->netdev->dev_addr,
1905                                                   I40E_VLAN_ANY,
1906                                                   is_vf, is_netdev)) {
1907                         i40e_del_filter(vsi, vsi->netdev->dev_addr,
1908                                         I40E_VLAN_ANY, is_vf, is_netdev);
1909                         add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, 0,
1910                                                 is_vf, is_netdev);
1911                         if (!add_f) {
1912                                 dev_info(&vsi->back->pdev->dev,
1913                                          "Could not add filter 0 for %pM\n",
1914                                          vsi->netdev->dev_addr);
1915                                 return -ENOMEM;
1916                         }
1917                 }
1918         }
1919
1920         /* Do not assume that I40E_VLAN_ANY should be reset to VLAN 0 */
1921         if (vid > 0 && !vsi->info.pvid) {
1922                 list_for_each_entry(f, &vsi->mac_filter_list, list) {
1923                         if (i40e_find_filter(vsi, f->macaddr, I40E_VLAN_ANY,
1924                                              is_vf, is_netdev)) {
1925                                 i40e_del_filter(vsi, f->macaddr, I40E_VLAN_ANY,
1926                                                 is_vf, is_netdev);
1927                                 add_f = i40e_add_filter(vsi, f->macaddr,
1928                                                         0, is_vf, is_netdev);
1929                                 if (!add_f) {
1930                                         dev_info(&vsi->back->pdev->dev,
1931                                                  "Could not add filter 0 for %pM\n",
1932                                                  f->macaddr);
1933                                         return -ENOMEM;
1934                                 }
1935                         }
1936                 }
1937         }
1938
1939         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1940             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1941                 return 0;
1942
1943         return i40e_sync_vsi_filters(vsi);
1944 }
1945
1946 /**
1947  * i40e_vsi_kill_vlan - Remove vsi membership for given vlan
1948  * @vsi: the vsi being configured
1949  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
1950  *
1951  * Return: 0 on success or negative otherwise
1952  **/
1953 int i40e_vsi_kill_vlan(struct i40e_vsi *vsi, s16 vid)
1954 {
1955         struct net_device *netdev = vsi->netdev;
1956         struct i40e_mac_filter *f, *add_f;
1957         bool is_vf, is_netdev;
1958         int filter_count = 0;
1959
1960         is_vf = (vsi->type == I40E_VSI_SRIOV);
1961         is_netdev = !!(netdev);
1962
1963         if (is_netdev)
1964                 i40e_del_filter(vsi, netdev->dev_addr, vid, is_vf, is_netdev);
1965
1966         list_for_each_entry(f, &vsi->mac_filter_list, list)
1967                 i40e_del_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
1968
1969         /* go through all the filters for this VSI and if there is only
1970          * vid == 0 it means there are no other filters, so vid 0 must
1971          * be replaced with -1. This signifies that we should from now
1972          * on accept any traffic (with any tag present, or untagged)
1973          */
1974         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1975                 if (is_netdev) {
1976                         if (f->vlan &&
1977                             ether_addr_equal(netdev->dev_addr, f->macaddr))
1978                                 filter_count++;
1979                 }
1980
1981                 if (f->vlan)
1982                         filter_count++;
1983         }
1984
1985         if (!filter_count && is_netdev) {
1986                 i40e_del_filter(vsi, netdev->dev_addr, 0, is_vf, is_netdev);
1987                 f = i40e_add_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY,
1988                                     is_vf, is_netdev);
1989                 if (!f) {
1990                         dev_info(&vsi->back->pdev->dev,
1991                                  "Could not add filter %d for %pM\n",
1992                                  I40E_VLAN_ANY, netdev->dev_addr);
1993                         return -ENOMEM;
1994                 }
1995         }
1996
1997         if (!filter_count) {
1998                 list_for_each_entry(f, &vsi->mac_filter_list, list) {
1999                         i40e_del_filter(vsi, f->macaddr, 0, is_vf, is_netdev);
2000                         add_f = i40e_add_filter(vsi, f->macaddr, I40E_VLAN_ANY,
2001                                             is_vf, is_netdev);
2002                         if (!add_f) {
2003                                 dev_info(&vsi->back->pdev->dev,
2004                                          "Could not add filter %d for %pM\n",
2005                                          I40E_VLAN_ANY, f->macaddr);
2006                                 return -ENOMEM;
2007                         }
2008                 }
2009         }
2010
2011         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
2012             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
2013                 return 0;
2014
2015         return i40e_sync_vsi_filters(vsi);
2016 }
2017
2018 /**
2019  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2020  * @netdev: network interface to be adjusted
2021  * @vid: vlan id to be added
2022  *
2023  * net_device_ops implementation for adding vlan ids
2024  **/
2025 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2026                                 __always_unused __be16 proto, u16 vid)
2027 {
2028         struct i40e_netdev_priv *np = netdev_priv(netdev);
2029         struct i40e_vsi *vsi = np->vsi;
2030         int ret = 0;
2031
2032         if (vid > 4095)
2033                 return -EINVAL;
2034
2035         netdev_info(netdev, "adding %pM vid=%d\n", netdev->dev_addr, vid);
2036
2037         /* If the network stack called us with vid = 0 then
2038          * it is asking to receive priority tagged packets with
2039          * vlan id 0.  Our HW receives them by default when configured
2040          * to receive untagged packets so there is no need to add an
2041          * extra filter for vlan 0 tagged packets.
2042          */
2043         if (vid)
2044                 ret = i40e_vsi_add_vlan(vsi, vid);
2045
2046         if (!ret && (vid < VLAN_N_VID))
2047                 set_bit(vid, vsi->active_vlans);
2048
2049         return ret;
2050 }
2051
2052 /**
2053  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2054  * @netdev: network interface to be adjusted
2055  * @vid: vlan id to be removed
2056  *
2057  * net_device_ops implementation for removing vlan ids
2058  **/
2059 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2060                                  __always_unused __be16 proto, u16 vid)
2061 {
2062         struct i40e_netdev_priv *np = netdev_priv(netdev);
2063         struct i40e_vsi *vsi = np->vsi;
2064
2065         netdev_info(netdev, "removing %pM vid=%d\n", netdev->dev_addr, vid);
2066
2067         /* return code is ignored as there is nothing a user
2068          * can do about failure to remove and a log message was
2069          * already printed from the other function
2070          */
2071         i40e_vsi_kill_vlan(vsi, vid);
2072
2073         clear_bit(vid, vsi->active_vlans);
2074
2075         return 0;
2076 }
2077
2078 /**
2079  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2080  * @vsi: the vsi being brought back up
2081  **/
2082 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2083 {
2084         u16 vid;
2085
2086         if (!vsi->netdev)
2087                 return;
2088
2089         i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
2090
2091         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2092                 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
2093                                      vid);
2094 }
2095
2096 /**
2097  * i40e_vsi_add_pvid - Add pvid for the VSI
2098  * @vsi: the vsi being adjusted
2099  * @vid: the vlan id to set as a PVID
2100  **/
2101 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2102 {
2103         struct i40e_vsi_context ctxt;
2104         i40e_status aq_ret;
2105
2106         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2107         vsi->info.pvid = cpu_to_le16(vid);
2108         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2109                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2110                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2111
2112         ctxt.seid = vsi->seid;
2113         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
2114         aq_ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2115         if (aq_ret) {
2116                 dev_info(&vsi->back->pdev->dev,
2117                          "%s: update vsi failed, aq_err=%d\n",
2118                          __func__, vsi->back->hw.aq.asq_last_status);
2119                 return -ENOENT;
2120         }
2121
2122         return 0;
2123 }
2124
2125 /**
2126  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
2127  * @vsi: the vsi being adjusted
2128  *
2129  * Just use the vlan_rx_register() service to put it back to normal
2130  **/
2131 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2132 {
2133         i40e_vlan_stripping_disable(vsi);
2134
2135         vsi->info.pvid = 0;
2136 }
2137
2138 /**
2139  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
2140  * @vsi: ptr to the VSI
2141  *
2142  * If this function returns with an error, then it's possible one or
2143  * more of the rings is populated (while the rest are not).  It is the
2144  * callers duty to clean those orphaned rings.
2145  *
2146  * Return 0 on success, negative on failure
2147  **/
2148 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2149 {
2150         int i, err = 0;
2151
2152         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2153                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2154
2155         return err;
2156 }
2157
2158 /**
2159  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
2160  * @vsi: ptr to the VSI
2161  *
2162  * Free VSI's transmit software resources
2163  **/
2164 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
2165 {
2166         int i;
2167
2168         if (!vsi->tx_rings)
2169                 return;
2170
2171         for (i = 0; i < vsi->num_queue_pairs; i++)
2172                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
2173                         i40e_free_tx_resources(vsi->tx_rings[i]);
2174 }
2175
2176 /**
2177  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
2178  * @vsi: ptr to the VSI
2179  *
2180  * If this function returns with an error, then it's possible one or
2181  * more of the rings is populated (while the rest are not).  It is the
2182  * callers duty to clean those orphaned rings.
2183  *
2184  * Return 0 on success, negative on failure
2185  **/
2186 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
2187 {
2188         int i, err = 0;
2189
2190         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2191                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
2192         return err;
2193 }
2194
2195 /**
2196  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
2197  * @vsi: ptr to the VSI
2198  *
2199  * Free all receive software resources
2200  **/
2201 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
2202 {
2203         int i;
2204
2205         if (!vsi->rx_rings)
2206                 return;
2207
2208         for (i = 0; i < vsi->num_queue_pairs; i++)
2209                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
2210                         i40e_free_rx_resources(vsi->rx_rings[i]);
2211 }
2212
2213 /**
2214  * i40e_configure_tx_ring - Configure a transmit ring context and rest
2215  * @ring: The Tx ring to configure
2216  *
2217  * Configure the Tx descriptor ring in the HMC context.
2218  **/
2219 static int i40e_configure_tx_ring(struct i40e_ring *ring)
2220 {
2221         struct i40e_vsi *vsi = ring->vsi;
2222         u16 pf_q = vsi->base_queue + ring->queue_index;
2223         struct i40e_hw *hw = &vsi->back->hw;
2224         struct i40e_hmc_obj_txq tx_ctx;
2225         i40e_status err = 0;
2226         u32 qtx_ctl = 0;
2227
2228         /* some ATR related tx ring init */
2229         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
2230                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
2231                 ring->atr_count = 0;
2232         } else {
2233                 ring->atr_sample_rate = 0;
2234         }
2235
2236         /* initialize XPS */
2237         if (ring->q_vector && ring->netdev &&
2238             vsi->tc_config.numtc <= 1 &&
2239             !test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state))
2240                 netif_set_xps_queue(ring->netdev,
2241                                     &ring->q_vector->affinity_mask,
2242                                     ring->queue_index);
2243
2244         /* clear the context structure first */
2245         memset(&tx_ctx, 0, sizeof(tx_ctx));
2246
2247         tx_ctx.new_context = 1;
2248         tx_ctx.base = (ring->dma / 128);
2249         tx_ctx.qlen = ring->count;
2250         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
2251                                                I40E_FLAG_FD_ATR_ENABLED));
2252         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
2253         /* FDIR VSI tx ring can still use RS bit and writebacks */
2254         if (vsi->type != I40E_VSI_FDIR)
2255                 tx_ctx.head_wb_ena = 1;
2256         tx_ctx.head_wb_addr = ring->dma +
2257                               (ring->count * sizeof(struct i40e_tx_desc));
2258
2259         /* As part of VSI creation/update, FW allocates certain
2260          * Tx arbitration queue sets for each TC enabled for
2261          * the VSI. The FW returns the handles to these queue
2262          * sets as part of the response buffer to Add VSI,
2263          * Update VSI, etc. AQ commands. It is expected that
2264          * these queue set handles be associated with the Tx
2265          * queues by the driver as part of the TX queue context
2266          * initialization. This has to be done regardless of
2267          * DCB as by default everything is mapped to TC0.
2268          */
2269         tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
2270         tx_ctx.rdylist_act = 0;
2271
2272         /* clear the context in the HMC */
2273         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2274         if (err) {
2275                 dev_info(&vsi->back->pdev->dev,
2276                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
2277                          ring->queue_index, pf_q, err);
2278                 return -ENOMEM;
2279         }
2280
2281         /* set the context in the HMC */
2282         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2283         if (err) {
2284                 dev_info(&vsi->back->pdev->dev,
2285                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
2286                          ring->queue_index, pf_q, err);
2287                 return -ENOMEM;
2288         }
2289
2290         /* Now associate this queue with this PCI function */
2291         if (vsi->type == I40E_VSI_VMDQ2)
2292                 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
2293         else
2294                 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2295         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2296                     I40E_QTX_CTL_PF_INDX_MASK);
2297         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
2298         i40e_flush(hw);
2299
2300         clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);
2301
2302         /* cache tail off for easier writes later */
2303         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
2304
2305         return 0;
2306 }
2307
2308 /**
2309  * i40e_configure_rx_ring - Configure a receive ring context
2310  * @ring: The Rx ring to configure
2311  *
2312  * Configure the Rx descriptor ring in the HMC context.
2313  **/
2314 static int i40e_configure_rx_ring(struct i40e_ring *ring)
2315 {
2316         struct i40e_vsi *vsi = ring->vsi;
2317         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
2318         u16 pf_q = vsi->base_queue + ring->queue_index;
2319         struct i40e_hw *hw = &vsi->back->hw;
2320         struct i40e_hmc_obj_rxq rx_ctx;
2321         i40e_status err = 0;
2322
2323         ring->state = 0;
2324
2325         /* clear the context structure first */
2326         memset(&rx_ctx, 0, sizeof(rx_ctx));
2327
2328         ring->rx_buf_len = vsi->rx_buf_len;
2329         ring->rx_hdr_len = vsi->rx_hdr_len;
2330
2331         rx_ctx.dbuff = ring->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT;
2332         rx_ctx.hbuff = ring->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT;
2333
2334         rx_ctx.base = (ring->dma / 128);
2335         rx_ctx.qlen = ring->count;
2336
2337         if (vsi->back->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED) {
2338                 set_ring_16byte_desc_enabled(ring);
2339                 rx_ctx.dsize = 0;
2340         } else {
2341                 rx_ctx.dsize = 1;
2342         }
2343
2344         rx_ctx.dtype = vsi->dtype;
2345         if (vsi->dtype) {
2346                 set_ring_ps_enabled(ring);
2347                 rx_ctx.hsplit_0 = I40E_RX_SPLIT_L2      |
2348                                   I40E_RX_SPLIT_IP      |
2349                                   I40E_RX_SPLIT_TCP_UDP |
2350                                   I40E_RX_SPLIT_SCTP;
2351         } else {
2352                 rx_ctx.hsplit_0 = 0;
2353         }
2354
2355         rx_ctx.rxmax = min_t(u16, vsi->max_frame,
2356                                   (chain_len * ring->rx_buf_len));
2357         rx_ctx.tphrdesc_ena = 1;
2358         rx_ctx.tphwdesc_ena = 1;
2359         rx_ctx.tphdata_ena = 1;
2360         rx_ctx.tphhead_ena = 1;
2361         if (hw->revision_id == 0)
2362                 rx_ctx.lrxqthresh = 0;
2363         else
2364                 rx_ctx.lrxqthresh = 2;
2365         rx_ctx.crcstrip = 1;
2366         rx_ctx.l2tsel = 1;
2367         rx_ctx.showiv = 1;
2368         /* set the prefena field to 1 because the manual says to */
2369         rx_ctx.prefena = 1;
2370
2371         /* clear the context in the HMC */
2372         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
2373         if (err) {
2374                 dev_info(&vsi->back->pdev->dev,
2375                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
2376                          ring->queue_index, pf_q, err);
2377                 return -ENOMEM;
2378         }
2379
2380         /* set the context in the HMC */
2381         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
2382         if (err) {
2383                 dev_info(&vsi->back->pdev->dev,
2384                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
2385                          ring->queue_index, pf_q, err);
2386                 return -ENOMEM;
2387         }
2388
2389         /* cache tail for quicker writes, and clear the reg before use */
2390         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
2391         writel(0, ring->tail);
2392
2393         i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
2394
2395         return 0;
2396 }
2397
2398 /**
2399  * i40e_vsi_configure_tx - Configure the VSI for Tx
2400  * @vsi: VSI structure describing this set of rings and resources
2401  *
2402  * Configure the Tx VSI for operation.
2403  **/
2404 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
2405 {
2406         int err = 0;
2407         u16 i;
2408
2409         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
2410                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
2411
2412         return err;
2413 }
2414
2415 /**
2416  * i40e_vsi_configure_rx - Configure the VSI for Rx
2417  * @vsi: the VSI being configured
2418  *
2419  * Configure the Rx VSI for operation.
2420  **/
2421 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
2422 {
2423         int err = 0;
2424         u16 i;
2425
2426         if (vsi->netdev && (vsi->netdev->mtu > ETH_DATA_LEN))
2427                 vsi->max_frame = vsi->netdev->mtu + ETH_HLEN
2428                                + ETH_FCS_LEN + VLAN_HLEN;
2429         else
2430                 vsi->max_frame = I40E_RXBUFFER_2048;
2431
2432         /* figure out correct receive buffer length */
2433         switch (vsi->back->flags & (I40E_FLAG_RX_1BUF_ENABLED |
2434                                     I40E_FLAG_RX_PS_ENABLED)) {
2435         case I40E_FLAG_RX_1BUF_ENABLED:
2436                 vsi->rx_hdr_len = 0;
2437                 vsi->rx_buf_len = vsi->max_frame;
2438                 vsi->dtype = I40E_RX_DTYPE_NO_SPLIT;
2439                 break;
2440         case I40E_FLAG_RX_PS_ENABLED:
2441                 vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
2442                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
2443                 vsi->dtype = I40E_RX_DTYPE_HEADER_SPLIT;
2444                 break;
2445         default:
2446                 vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
2447                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
2448                 vsi->dtype = I40E_RX_DTYPE_SPLIT_ALWAYS;
2449                 break;
2450         }
2451
2452         /* round up for the chip's needs */
2453         vsi->rx_hdr_len = ALIGN(vsi->rx_hdr_len,
2454                                 (1 << I40E_RXQ_CTX_HBUFF_SHIFT));
2455         vsi->rx_buf_len = ALIGN(vsi->rx_buf_len,
2456                                 (1 << I40E_RXQ_CTX_DBUFF_SHIFT));
2457
2458         /* set up individual rings */
2459         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2460                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
2461
2462         return err;
2463 }
2464
2465 /**
2466  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
2467  * @vsi: ptr to the VSI
2468  **/
2469 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
2470 {
2471         struct i40e_ring *tx_ring, *rx_ring;
2472         u16 qoffset, qcount;
2473         int i, n;
2474
2475         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED))
2476                 return;
2477
2478         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
2479                 if (!(vsi->tc_config.enabled_tc & (1 << n)))
2480                         continue;
2481
2482                 qoffset = vsi->tc_config.tc_info[n].qoffset;
2483                 qcount = vsi->tc_config.tc_info[n].qcount;
2484                 for (i = qoffset; i < (qoffset + qcount); i++) {
2485                         rx_ring = vsi->rx_rings[i];
2486                         tx_ring = vsi->tx_rings[i];
2487                         rx_ring->dcb_tc = n;
2488                         tx_ring->dcb_tc = n;
2489                 }
2490         }
2491 }
2492
2493 /**
2494  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
2495  * @vsi: ptr to the VSI
2496  **/
2497 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
2498 {
2499         if (vsi->netdev)
2500                 i40e_set_rx_mode(vsi->netdev);
2501 }
2502
2503 /**
2504  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
2505  * @vsi: Pointer to the targeted VSI
2506  *
2507  * This function replays the hlist on the hw where all the SB Flow Director
2508  * filters were saved.
2509  **/
2510 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
2511 {
2512         struct i40e_fdir_filter *filter;
2513         struct i40e_pf *pf = vsi->back;
2514         struct hlist_node *node;
2515
2516         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
2517                 return;
2518
2519         hlist_for_each_entry_safe(filter, node,
2520                                   &pf->fdir_filter_list, fdir_node) {
2521                 i40e_add_del_fdir(vsi, filter, true);
2522         }
2523 }
2524
2525 /**
2526  * i40e_vsi_configure - Set up the VSI for action
2527  * @vsi: the VSI being configured
2528  **/
2529 static int i40e_vsi_configure(struct i40e_vsi *vsi)
2530 {
2531         int err;
2532
2533         i40e_set_vsi_rx_mode(vsi);
2534         i40e_restore_vlan(vsi);
2535         i40e_vsi_config_dcb_rings(vsi);
2536         err = i40e_vsi_configure_tx(vsi);
2537         if (!err)
2538                 err = i40e_vsi_configure_rx(vsi);
2539
2540         return err;
2541 }
2542
2543 /**
2544  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
2545  * @vsi: the VSI being configured
2546  **/
2547 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
2548 {
2549         struct i40e_pf *pf = vsi->back;
2550         struct i40e_q_vector *q_vector;
2551         struct i40e_hw *hw = &pf->hw;
2552         u16 vector;
2553         int i, q;
2554         u32 val;
2555         u32 qp;
2556
2557         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
2558          * and PFINT_LNKLSTn registers, e.g.:
2559          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
2560          */
2561         qp = vsi->base_queue;
2562         vector = vsi->base_vector;
2563         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
2564                 q_vector = vsi->q_vectors[i];
2565                 q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
2566                 q_vector->rx.latency_range = I40E_LOW_LATENCY;
2567                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
2568                      q_vector->rx.itr);
2569                 q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
2570                 q_vector->tx.latency_range = I40E_LOW_LATENCY;
2571                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
2572                      q_vector->tx.itr);
2573
2574                 /* Linked list for the queuepairs assigned to this vector */
2575                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
2576                 for (q = 0; q < q_vector->num_ringpairs; q++) {
2577                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2578                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
2579                               (vector      << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2580                               (qp          << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
2581                               (I40E_QUEUE_TYPE_TX
2582                                       << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2583
2584                         wr32(hw, I40E_QINT_RQCTL(qp), val);
2585
2586                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2587                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)  |
2588                               (vector      << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2589                               ((qp+1)      << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)|
2590                               (I40E_QUEUE_TYPE_RX
2591                                       << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2592
2593                         /* Terminate the linked list */
2594                         if (q == (q_vector->num_ringpairs - 1))
2595                                 val |= (I40E_QUEUE_END_OF_LIST
2596                                            << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2597
2598                         wr32(hw, I40E_QINT_TQCTL(qp), val);
2599                         qp++;
2600                 }
2601         }
2602
2603         i40e_flush(hw);
2604 }
2605
2606 /**
2607  * i40e_enable_misc_int_causes - enable the non-queue interrupts
2608  * @hw: ptr to the hardware info
2609  **/
2610 static void i40e_enable_misc_int_causes(struct i40e_hw *hw)
2611 {
2612         u32 val;
2613
2614         /* clear things first */
2615         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2616         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2617
2618         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
2619               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
2620               I40E_PFINT_ICR0_ENA_GRST_MASK          |
2621               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
2622               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
2623               I40E_PFINT_ICR0_ENA_TIMESYNC_MASK      |
2624               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
2625               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
2626               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
2627
2628         wr32(hw, I40E_PFINT_ICR0_ENA, val);
2629
2630         /* SW_ITR_IDX = 0, but don't change INTENA */
2631         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2632                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2633
2634         /* OTHER_ITR_IDX = 0 */
2635         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2636 }
2637
2638 /**
2639  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
2640  * @vsi: the VSI being configured
2641  **/
2642 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
2643 {
2644         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
2645         struct i40e_pf *pf = vsi->back;
2646         struct i40e_hw *hw = &pf->hw;
2647         u32 val;
2648
2649         /* set the ITR configuration */
2650         q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
2651         q_vector->rx.latency_range = I40E_LOW_LATENCY;
2652         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
2653         q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
2654         q_vector->tx.latency_range = I40E_LOW_LATENCY;
2655         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
2656
2657         i40e_enable_misc_int_causes(hw);
2658
2659         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2660         wr32(hw, I40E_PFINT_LNKLST0, 0);
2661
2662         /* Associate the queue pair to the vector and enable the queue int */
2663         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                  |
2664               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2665               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2666
2667         wr32(hw, I40E_QINT_RQCTL(0), val);
2668
2669         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
2670               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2671               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2672
2673         wr32(hw, I40E_QINT_TQCTL(0), val);
2674         i40e_flush(hw);
2675 }
2676
2677 /**
2678  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
2679  * @pf: board private structure
2680  **/
2681 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
2682 {
2683         struct i40e_hw *hw = &pf->hw;
2684
2685         wr32(hw, I40E_PFINT_DYN_CTL0,
2686              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
2687         i40e_flush(hw);
2688 }
2689
2690 /**
2691  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
2692  * @pf: board private structure
2693  **/
2694 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
2695 {
2696         struct i40e_hw *hw = &pf->hw;
2697         u32 val;
2698
2699         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
2700               I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
2701               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
2702
2703         wr32(hw, I40E_PFINT_DYN_CTL0, val);
2704         i40e_flush(hw);
2705 }
2706
2707 /**
2708  * i40e_irq_dynamic_enable - Enable default interrupt generation settings
2709  * @vsi: pointer to a vsi
2710  * @vector: enable a particular Hw Interrupt vector
2711  **/
2712 void i40e_irq_dynamic_enable(struct i40e_vsi *vsi, int vector)
2713 {
2714         struct i40e_pf *pf = vsi->back;
2715         struct i40e_hw *hw = &pf->hw;
2716         u32 val;
2717
2718         val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
2719               I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
2720               (I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
2721         wr32(hw, I40E_PFINT_DYN_CTLN(vector - 1), val);
2722         /* skip the flush */
2723 }
2724
2725 /**
2726  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
2727  * @irq: interrupt number
2728  * @data: pointer to a q_vector
2729  **/
2730 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
2731 {
2732         struct i40e_q_vector *q_vector = data;
2733
2734         if (!q_vector->tx.ring && !q_vector->rx.ring)
2735                 return IRQ_HANDLED;
2736
2737         napi_schedule(&q_vector->napi);
2738
2739         return IRQ_HANDLED;
2740 }
2741
2742 /**
2743  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
2744  * @vsi: the VSI being configured
2745  * @basename: name for the vector
2746  *
2747  * Allocates MSI-X vectors and requests interrupts from the kernel.
2748  **/
2749 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
2750 {
2751         int q_vectors = vsi->num_q_vectors;
2752         struct i40e_pf *pf = vsi->back;
2753         int base = vsi->base_vector;
2754         int rx_int_idx = 0;
2755         int tx_int_idx = 0;
2756         int vector, err;
2757
2758         for (vector = 0; vector < q_vectors; vector++) {
2759                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
2760
2761                 if (q_vector->tx.ring && q_vector->rx.ring) {
2762                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2763                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
2764                         tx_int_idx++;
2765                 } else if (q_vector->rx.ring) {
2766                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2767                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
2768                 } else if (q_vector->tx.ring) {
2769                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2770                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
2771                 } else {
2772                         /* skip this unused q_vector */
2773                         continue;
2774                 }
2775                 err = request_irq(pf->msix_entries[base + vector].vector,
2776                                   vsi->irq_handler,
2777                                   0,
2778                                   q_vector->name,
2779                                   q_vector);
2780                 if (err) {
2781                         dev_info(&pf->pdev->dev,
2782                                  "%s: request_irq failed, error: %d\n",
2783                                  __func__, err);
2784                         goto free_queue_irqs;
2785                 }
2786                 /* assign the mask for this irq */
2787                 irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
2788                                       &q_vector->affinity_mask);
2789         }
2790
2791         vsi->irqs_ready = true;
2792         return 0;
2793
2794 free_queue_irqs:
2795         while (vector) {
2796                 vector--;
2797                 irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
2798                                       NULL);
2799                 free_irq(pf->msix_entries[base + vector].vector,
2800                          &(vsi->q_vectors[vector]));
2801         }
2802         return err;
2803 }
2804
2805 /**
2806  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
2807  * @vsi: the VSI being un-configured
2808  **/
2809 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
2810 {
2811         struct i40e_pf *pf = vsi->back;
2812         struct i40e_hw *hw = &pf->hw;
2813         int base = vsi->base_vector;
2814         int i;
2815
2816         for (i = 0; i < vsi->num_queue_pairs; i++) {
2817                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0);
2818                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0);
2819         }
2820
2821         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
2822                 for (i = vsi->base_vector;
2823                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
2824                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
2825
2826                 i40e_flush(hw);
2827                 for (i = 0; i < vsi->num_q_vectors; i++)
2828                         synchronize_irq(pf->msix_entries[i + base].vector);
2829         } else {
2830                 /* Legacy and MSI mode - this stops all interrupt handling */
2831                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
2832                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
2833                 i40e_flush(hw);
2834                 synchronize_irq(pf->pdev->irq);
2835         }
2836 }
2837
2838 /**
2839  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
2840  * @vsi: the VSI being configured
2841  **/
2842 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
2843 {
2844         struct i40e_pf *pf = vsi->back;
2845         int i;
2846
2847         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
2848                 for (i = vsi->base_vector;
2849                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
2850                         i40e_irq_dynamic_enable(vsi, i);
2851         } else {
2852                 i40e_irq_dynamic_enable_icr0(pf);
2853         }
2854
2855         i40e_flush(&pf->hw);
2856         return 0;
2857 }
2858
2859 /**
2860  * i40e_stop_misc_vector - Stop the vector that handles non-queue events
2861  * @pf: board private structure
2862  **/
2863 static void i40e_stop_misc_vector(struct i40e_pf *pf)
2864 {
2865         /* Disable ICR 0 */
2866         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
2867         i40e_flush(&pf->hw);
2868 }
2869
2870 /**
2871  * i40e_intr - MSI/Legacy and non-queue interrupt handler
2872  * @irq: interrupt number
2873  * @data: pointer to a q_vector
2874  *
2875  * This is the handler used for all MSI/Legacy interrupts, and deals
2876  * with both queue and non-queue interrupts.  This is also used in
2877  * MSIX mode to handle the non-queue interrupts.
2878  **/
2879 static irqreturn_t i40e_intr(int irq, void *data)
2880 {
2881         struct i40e_pf *pf = (struct i40e_pf *)data;
2882         struct i40e_hw *hw = &pf->hw;
2883         irqreturn_t ret = IRQ_NONE;
2884         u32 icr0, icr0_remaining;
2885         u32 val, ena_mask;
2886
2887         icr0 = rd32(hw, I40E_PFINT_ICR0);
2888         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
2889
2890         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
2891         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
2892                 goto enable_intr;
2893
2894         /* if interrupt but no bits showing, must be SWINT */
2895         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
2896             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
2897                 pf->sw_int_count++;
2898
2899         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
2900         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
2901
2902                 /* temporarily disable queue cause for NAPI processing */
2903                 u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
2904                 qval &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
2905                 wr32(hw, I40E_QINT_RQCTL(0), qval);
2906
2907                 qval = rd32(hw, I40E_QINT_TQCTL(0));
2908                 qval &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
2909                 wr32(hw, I40E_QINT_TQCTL(0), qval);
2910
2911                 if (!test_bit(__I40E_DOWN, &pf->state))
2912                         napi_schedule(&pf->vsi[pf->lan_vsi]->q_vectors[0]->napi);
2913         }
2914
2915         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
2916                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
2917                 set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
2918         }
2919
2920         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
2921                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
2922                 set_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
2923         }
2924
2925         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
2926                 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
2927                 set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state);
2928         }
2929
2930         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
2931                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
2932                         set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
2933                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
2934                 val = rd32(hw, I40E_GLGEN_RSTAT);
2935                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
2936                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
2937                 if (val == I40E_RESET_CORER) {
2938                         pf->corer_count++;
2939                 } else if (val == I40E_RESET_GLOBR) {
2940                         pf->globr_count++;
2941                 } else if (val == I40E_RESET_EMPR) {
2942                         pf->empr_count++;
2943                         set_bit(__I40E_EMP_RESET_REQUESTED, &pf->state);
2944                 }
2945         }
2946
2947         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
2948                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
2949                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
2950         }
2951
2952         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
2953                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
2954
2955                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
2956                         icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
2957                         i40e_ptp_tx_hwtstamp(pf);
2958                 }
2959         }
2960
2961         /* If a critical error is pending we have no choice but to reset the
2962          * device.
2963          * Report and mask out any remaining unexpected interrupts.
2964          */
2965         icr0_remaining = icr0 & ena_mask;
2966         if (icr0_remaining) {
2967                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
2968                          icr0_remaining);
2969                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
2970                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
2971                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
2972                         dev_info(&pf->pdev->dev, "device will be reset\n");
2973                         set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
2974                         i40e_service_event_schedule(pf);
2975                 }
2976                 ena_mask &= ~icr0_remaining;
2977         }
2978         ret = IRQ_HANDLED;
2979
2980 enable_intr:
2981         /* re-enable interrupt causes */
2982         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
2983         if (!test_bit(__I40E_DOWN, &pf->state)) {
2984                 i40e_service_event_schedule(pf);
2985                 i40e_irq_dynamic_enable_icr0(pf);
2986         }
2987
2988         return ret;
2989 }
2990
2991 /**
2992  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
2993  * @tx_ring:  tx ring to clean
2994  * @budget:   how many cleans we're allowed
2995  *
2996  * Returns true if there's any budget left (e.g. the clean is finished)
2997  **/
2998 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
2999 {
3000         struct i40e_vsi *vsi = tx_ring->vsi;
3001         u16 i = tx_ring->next_to_clean;
3002         struct i40e_tx_buffer *tx_buf;
3003         struct i40e_tx_desc *tx_desc;
3004
3005         tx_buf = &tx_ring->tx_bi[i];
3006         tx_desc = I40E_TX_DESC(tx_ring, i);
3007         i -= tx_ring->count;
3008
3009         do {
3010                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
3011
3012                 /* if next_to_watch is not set then there is no work pending */
3013                 if (!eop_desc)
3014                         break;
3015
3016                 /* prevent any other reads prior to eop_desc */
3017                 read_barrier_depends();
3018
3019                 /* if the descriptor isn't done, no work yet to do */
3020                 if (!(eop_desc->cmd_type_offset_bsz &
3021                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
3022                         break;
3023
3024                 /* clear next_to_watch to prevent false hangs */
3025                 tx_buf->next_to_watch = NULL;
3026
3027                 /* unmap skb header data */
3028                 dma_unmap_single(tx_ring->dev,
3029                                  dma_unmap_addr(tx_buf, dma),
3030                                  dma_unmap_len(tx_buf, len),
3031                                  DMA_TO_DEVICE);
3032
3033                 dma_unmap_len_set(tx_buf, len, 0);
3034
3035
3036                 /* move to the next desc and buffer to clean */
3037                 tx_buf++;
3038                 tx_desc++;
3039                 i++;
3040                 if (unlikely(!i)) {
3041                         i -= tx_ring->count;
3042                         tx_buf = tx_ring->tx_bi;
3043                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3044                 }
3045
3046                 /* update budget accounting */
3047                 budget--;
3048         } while (likely(budget));
3049
3050         i += tx_ring->count;
3051         tx_ring->next_to_clean = i;
3052
3053         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
3054                 i40e_irq_dynamic_enable(vsi,
3055                                 tx_ring->q_vector->v_idx + vsi->base_vector);
3056         }
3057         return budget > 0;
3058 }
3059
3060 /**
3061  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
3062  * @irq: interrupt number
3063  * @data: pointer to a q_vector
3064  **/
3065 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
3066 {
3067         struct i40e_q_vector *q_vector = data;
3068         struct i40e_vsi *vsi;
3069
3070         if (!q_vector->tx.ring)
3071                 return IRQ_HANDLED;
3072
3073         vsi = q_vector->tx.ring->vsi;
3074         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
3075
3076         return IRQ_HANDLED;
3077 }
3078
3079 /**
3080  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
3081  * @vsi: the VSI being configured
3082  * @v_idx: vector index
3083  * @qp_idx: queue pair index
3084  **/
3085 static void map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
3086 {
3087         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3088         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
3089         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
3090
3091         tx_ring->q_vector = q_vector;
3092         tx_ring->next = q_vector->tx.ring;
3093         q_vector->tx.ring = tx_ring;
3094         q_vector->tx.count++;
3095
3096         rx_ring->q_vector = q_vector;
3097         rx_ring->next = q_vector->rx.ring;
3098         q_vector->rx.ring = rx_ring;
3099         q_vector->rx.count++;
3100 }
3101
3102 /**
3103  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
3104  * @vsi: the VSI being configured
3105  *
3106  * This function maps descriptor rings to the queue-specific vectors
3107  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
3108  * one vector per queue pair, but on a constrained vector budget, we
3109  * group the queue pairs as "efficiently" as possible.
3110  **/
3111 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
3112 {
3113         int qp_remaining = vsi->num_queue_pairs;
3114         int q_vectors = vsi->num_q_vectors;
3115         int num_ringpairs;
3116         int v_start = 0;
3117         int qp_idx = 0;
3118
3119         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
3120          * group them so there are multiple queues per vector.
3121          */
3122         for (; v_start < q_vectors && qp_remaining; v_start++) {
3123                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
3124
3125                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
3126
3127                 q_vector->num_ringpairs = num_ringpairs;
3128
3129                 q_vector->rx.count = 0;
3130                 q_vector->tx.count = 0;
3131                 q_vector->rx.ring = NULL;
3132                 q_vector->tx.ring = NULL;
3133
3134                 while (num_ringpairs--) {
3135                         map_vector_to_qp(vsi, v_start, qp_idx);
3136                         qp_idx++;
3137                         qp_remaining--;
3138                 }
3139         }
3140 }
3141
3142 /**
3143  * i40e_vsi_request_irq - Request IRQ from the OS
3144  * @vsi: the VSI being configured
3145  * @basename: name for the vector
3146  **/
3147 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
3148 {
3149         struct i40e_pf *pf = vsi->back;
3150         int err;
3151
3152         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
3153                 err = i40e_vsi_request_irq_msix(vsi, basename);
3154         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
3155                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
3156                                   pf->misc_int_name, pf);
3157         else
3158                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
3159                                   pf->misc_int_name, pf);
3160
3161         if (err)
3162                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
3163
3164         return err;
3165 }
3166
3167 #ifdef CONFIG_NET_POLL_CONTROLLER
3168 /**
3169  * i40e_netpoll - A Polling 'interrupt'handler
3170  * @netdev: network interface device structure
3171  *
3172  * This is used by netconsole to send skbs without having to re-enable
3173  * interrupts.  It's not called while the normal interrupt routine is executing.
3174  **/
3175 static void i40e_netpoll(struct net_device *netdev)
3176 {
3177         struct i40e_netdev_priv *np = netdev_priv(netdev);
3178         struct i40e_vsi *vsi = np->vsi;
3179         struct i40e_pf *pf = vsi->back;
3180         int i;
3181
3182         /* if interface is down do nothing */
3183         if (test_bit(__I40E_DOWN, &vsi->state))
3184                 return;
3185
3186         pf->flags |= I40E_FLAG_IN_NETPOLL;
3187         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3188                 for (i = 0; i < vsi->num_q_vectors; i++)
3189                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
3190         } else {
3191                 i40e_intr(pf->pdev->irq, netdev);
3192         }
3193         pf->flags &= ~I40E_FLAG_IN_NETPOLL;
3194 }
3195 #endif
3196
3197 /**
3198  * i40e_vsi_control_tx - Start or stop a VSI's rings
3199  * @vsi: the VSI being configured
3200  * @enable: start or stop the rings
3201  **/
3202 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
3203 {
3204         struct i40e_pf *pf = vsi->back;
3205         struct i40e_hw *hw = &pf->hw;
3206         int i, j, pf_q;
3207         u32 tx_reg;
3208
3209         pf_q = vsi->base_queue;
3210         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
3211
3212                 /* warn the TX unit of coming changes */
3213                 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
3214                 if (!enable)
3215                         udelay(10);
3216
3217                 for (j = 0; j < 50; j++) {
3218                         tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
3219                         if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
3220                             ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
3221                                 break;
3222                         usleep_range(1000, 2000);
3223                 }
3224                 /* Skip if the queue is already in the requested state */
3225                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3226                         continue;
3227
3228                 /* turn on/off the queue */
3229                 if (enable) {
3230                         wr32(hw, I40E_QTX_HEAD(pf_q), 0);
3231                         tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
3232                 } else {
3233                         tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3234                 }
3235
3236                 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
3237
3238                 /* wait for the change to finish */
3239                 for (j = 0; j < 10; j++) {
3240                         tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
3241                         if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3242                                 break;
3243
3244                         udelay(10);
3245                 }
3246                 if (j >= 10) {
3247                         dev_info(&pf->pdev->dev, "Tx ring %d %sable timeout\n",
3248                                  pf_q, (enable ? "en" : "dis"));
3249                         return -ETIMEDOUT;
3250                 }
3251         }
3252
3253         if (hw->revision_id == 0)
3254                 mdelay(50);
3255
3256         return 0;
3257 }
3258
3259 /**
3260  * i40e_vsi_control_rx - Start or stop a VSI's rings
3261  * @vsi: the VSI being configured
3262  * @enable: start or stop the rings
3263  **/
3264 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
3265 {
3266         struct i40e_pf *pf = vsi->back;
3267         struct i40e_hw *hw = &pf->hw;
3268         int i, j, pf_q;
3269         u32 rx_reg;
3270
3271         pf_q = vsi->base_queue;
3272         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
3273                 for (j = 0; j < 50; j++) {
3274                         rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
3275                         if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
3276                             ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
3277                                 break;
3278                         usleep_range(1000, 2000);
3279                 }
3280
3281                 /* Skip if the queue is already in the requested state */
3282                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3283                         continue;
3284
3285                 /* turn on/off the queue */
3286                 if (enable)
3287                         rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
3288                 else
3289                         rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
3290                 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
3291
3292                 /* wait for the change to finish */
3293                 for (j = 0; j < 10; j++) {
3294                         rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
3295
3296                         if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3297                                 break;
3298
3299                         udelay(10);
3300                 }
3301                 if (j >= 10) {
3302                         dev_info(&pf->pdev->dev, "Rx ring %d %sable timeout\n",
3303                                  pf_q, (enable ? "en" : "dis"));
3304                         return -ETIMEDOUT;
3305                 }
3306         }
3307
3308         return 0;
3309 }
3310
3311 /**
3312  * i40e_vsi_control_rings - Start or stop a VSI's rings
3313  * @vsi: the VSI being configured
3314  * @enable: start or stop the rings
3315  **/
3316 int i40e_vsi_control_rings(struct i40e_vsi *vsi, bool request)
3317 {
3318         int ret = 0;
3319
3320         /* do rx first for enable and last for disable */
3321         if (request) {
3322                 ret = i40e_vsi_control_rx(vsi, request);
3323                 if (ret)
3324                         return ret;
3325                 ret = i40e_vsi_control_tx(vsi, request);
3326         } else {
3327                 /* Ignore return value, we need to shutdown whatever we can */
3328                 i40e_vsi_control_tx(vsi, request);
3329                 i40e_vsi_control_rx(vsi, request);
3330         }
3331
3332         return ret;
3333 }
3334
3335 /**
3336  * i40e_vsi_free_irq - Free the irq association with the OS
3337  * @vsi: the VSI being configured
3338  **/
3339 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
3340 {
3341         struct i40e_pf *pf = vsi->back;
3342         struct i40e_hw *hw = &pf->hw;
3343         int base = vsi->base_vector;
3344         u32 val, qp;
3345         int i;
3346
3347         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3348                 if (!vsi->q_vectors)
3349                         return;
3350
3351                 if (!vsi->irqs_ready)
3352                         return;
3353
3354                 vsi->irqs_ready = false;
3355                 for (i = 0; i < vsi->num_q_vectors; i++) {
3356                         u16 vector = i + base;
3357
3358                         /* free only the irqs that were actually requested */
3359                         if (!vsi->q_vectors[i] ||
3360                             !vsi->q_vectors[i]->num_ringpairs)
3361                                 continue;
3362
3363                         /* clear the affinity_mask in the IRQ descriptor */
3364                         irq_set_affinity_hint(pf->msix_entries[vector].vector,
3365                                               NULL);
3366                         free_irq(pf->msix_entries[vector].vector,
3367                                  vsi->q_vectors[i]);
3368
3369                         /* Tear down the interrupt queue link list
3370                          *
3371                          * We know that they come in pairs and always
3372                          * the Rx first, then the Tx.  To clear the
3373                          * link list, stick the EOL value into the
3374                          * next_q field of the registers.
3375                          */
3376                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
3377                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
3378                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3379                         val |= I40E_QUEUE_END_OF_LIST
3380                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3381                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
3382
3383                         while (qp != I40E_QUEUE_END_OF_LIST) {
3384                                 u32 next;
3385
3386                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
3387
3388                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
3389                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
3390                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
3391                                          I40E_QINT_RQCTL_INTEVENT_MASK);
3392
3393                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
3394                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
3395
3396                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
3397
3398                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
3399
3400                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
3401                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
3402
3403                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
3404                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
3405                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
3406                                          I40E_QINT_TQCTL_INTEVENT_MASK);
3407
3408                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
3409                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
3410
3411                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
3412                                 qp = next;
3413                         }
3414                 }
3415         } else {
3416                 free_irq(pf->pdev->irq, pf);
3417
3418                 val = rd32(hw, I40E_PFINT_LNKLST0);
3419                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
3420                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3421                 val |= I40E_QUEUE_END_OF_LIST
3422                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
3423                 wr32(hw, I40E_PFINT_LNKLST0, val);
3424
3425                 val = rd32(hw, I40E_QINT_RQCTL(qp));
3426                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
3427                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
3428                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
3429                          I40E_QINT_RQCTL_INTEVENT_MASK);
3430
3431                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
3432                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
3433
3434                 wr32(hw, I40E_QINT_RQCTL(qp), val);
3435
3436                 val = rd32(hw, I40E_QINT_TQCTL(qp));
3437
3438                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
3439                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
3440                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
3441                          I40E_QINT_TQCTL_INTEVENT_MASK);
3442
3443                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
3444                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
3445
3446                 wr32(hw, I40E_QINT_TQCTL(qp), val);
3447         }
3448 }
3449
3450 /**
3451  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
3452  * @vsi: the VSI being configured
3453  * @v_idx: Index of vector to be freed
3454  *
3455  * This function frees the memory allocated to the q_vector.  In addition if
3456  * NAPI is enabled it will delete any references to the NAPI struct prior
3457  * to freeing the q_vector.
3458  **/
3459 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
3460 {
3461         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3462         struct i40e_ring *ring;
3463
3464         if (!q_vector)
3465                 return;
3466
3467         /* disassociate q_vector from rings */
3468         i40e_for_each_ring(ring, q_vector->tx)
3469                 ring->q_vector = NULL;
3470
3471         i40e_for_each_ring(ring, q_vector->rx)
3472                 ring->q_vector = NULL;
3473
3474         /* only VSI w/ an associated netdev is set up w/ NAPI */
3475         if (vsi->netdev)
3476                 netif_napi_del(&q_vector->napi);
3477
3478         vsi->q_vectors[v_idx] = NULL;
3479
3480         kfree_rcu(q_vector, rcu);
3481 }
3482
3483 /**
3484  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
3485  * @vsi: the VSI being un-configured
3486  *
3487  * This frees the memory allocated to the q_vectors and
3488  * deletes references to the NAPI struct.
3489  **/
3490 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
3491 {
3492         int v_idx;
3493
3494         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
3495                 i40e_free_q_vector(vsi, v_idx);
3496 }
3497
3498 /**
3499  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
3500  * @pf: board private structure
3501  **/
3502 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
3503 {
3504         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
3505         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3506                 pci_disable_msix(pf->pdev);
3507                 kfree(pf->msix_entries);
3508                 pf->msix_entries = NULL;
3509         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
3510                 pci_disable_msi(pf->pdev);
3511         }
3512         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
3513 }
3514
3515 /**
3516  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
3517  * @pf: board private structure
3518  *
3519  * We go through and clear interrupt specific resources and reset the structure
3520  * to pre-load conditions
3521  **/
3522 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
3523 {
3524         int i;
3525
3526         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
3527         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
3528                 if (pf->vsi[i])
3529                         i40e_vsi_free_q_vectors(pf->vsi[i]);
3530         i40e_reset_interrupt_capability(pf);
3531 }
3532
3533 /**
3534  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
3535  * @vsi: the VSI being configured
3536  **/
3537 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
3538 {
3539         int q_idx;
3540
3541         if (!vsi->netdev)
3542                 return;
3543
3544         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
3545                 napi_enable(&vsi->q_vectors[q_idx]->napi);
3546 }
3547
3548 /**
3549  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
3550  * @vsi: the VSI being configured
3551  **/
3552 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
3553 {
3554         int q_idx;
3555
3556         if (!vsi->netdev)
3557                 return;
3558
3559         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
3560                 napi_disable(&vsi->q_vectors[q_idx]->napi);
3561 }
3562
3563 /**
3564  * i40e_vsi_close - Shut down a VSI
3565  * @vsi: the vsi to be quelled
3566  **/
3567 static void i40e_vsi_close(struct i40e_vsi *vsi)
3568 {
3569         if (!test_and_set_bit(__I40E_DOWN, &vsi->state))
3570                 i40e_down(vsi);
3571         i40e_vsi_free_irq(vsi);
3572         i40e_vsi_free_tx_resources(vsi);
3573         i40e_vsi_free_rx_resources(vsi);
3574 }
3575
3576 /**
3577  * i40e_quiesce_vsi - Pause a given VSI
3578  * @vsi: the VSI being paused
3579  **/
3580 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
3581 {
3582         if (test_bit(__I40E_DOWN, &vsi->state))
3583                 return;
3584
3585         set_bit(__I40E_NEEDS_RESTART, &vsi->state);
3586         if (vsi->netdev && netif_running(vsi->netdev)) {
3587                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
3588         } else {
3589                 i40e_vsi_close(vsi);
3590         }
3591 }
3592
3593 /**
3594  * i40e_unquiesce_vsi - Resume a given VSI
3595  * @vsi: the VSI being resumed
3596  **/
3597 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
3598 {
3599         if (!test_bit(__I40E_NEEDS_RESTART, &vsi->state))
3600                 return;
3601
3602         clear_bit(__I40E_NEEDS_RESTART, &vsi->state);
3603         if (vsi->netdev && netif_running(vsi->netdev))
3604                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
3605         else
3606                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
3607 }
3608
3609 /**
3610  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
3611  * @pf: the PF
3612  **/
3613 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
3614 {
3615         int v;
3616
3617         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
3618                 if (pf->vsi[v])
3619                         i40e_quiesce_vsi(pf->vsi[v]);
3620         }
3621 }
3622
3623 /**
3624  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
3625  * @pf: the PF
3626  **/
3627 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
3628 {
3629         int v;
3630
3631         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
3632                 if (pf->vsi[v])
3633                         i40e_unquiesce_vsi(pf->vsi[v]);
3634         }
3635 }
3636
3637 /**
3638  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
3639  * @dcbcfg: the corresponding DCBx configuration structure
3640  *
3641  * Return the number of TCs from given DCBx configuration
3642  **/
3643 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
3644 {
3645         u8 num_tc = 0;
3646         int i;
3647
3648         /* Scan the ETS Config Priority Table to find
3649          * traffic class enabled for a given priority
3650          * and use the traffic class index to get the
3651          * number of traffic classes enabled
3652          */
3653         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
3654                 if (dcbcfg->etscfg.prioritytable[i] > num_tc)
3655                         num_tc = dcbcfg->etscfg.prioritytable[i];
3656         }
3657
3658         /* Traffic class index starts from zero so
3659          * increment to return the actual count
3660          */
3661         return num_tc + 1;
3662 }
3663
3664 /**
3665  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
3666  * @dcbcfg: the corresponding DCBx configuration structure
3667  *
3668  * Query the current DCB configuration and return the number of
3669  * traffic classes enabled from the given DCBX config
3670  **/
3671 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
3672 {
3673         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
3674         u8 enabled_tc = 1;
3675         u8 i;
3676
3677         for (i = 0; i < num_tc; i++)
3678                 enabled_tc |= 1 << i;
3679
3680         return enabled_tc;
3681 }
3682
3683 /**
3684  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
3685  * @pf: PF being queried
3686  *
3687  * Return number of traffic classes enabled for the given PF
3688  **/
3689 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
3690 {
3691         struct i40e_hw *hw = &pf->hw;
3692         u8 i, enabled_tc;
3693         u8 num_tc = 0;
3694         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
3695
3696         /* If DCB is not enabled then always in single TC */
3697         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
3698                 return 1;
3699
3700         /* MFP mode return count of enabled TCs for this PF */
3701         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
3702                 enabled_tc = pf->hw.func_caps.enabled_tcmap;
3703                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3704                         if (enabled_tc & (1 << i))
3705                                 num_tc++;
3706                 }
3707                 return num_tc;
3708         }
3709
3710         /* SFP mode will be enabled for all TCs on port */
3711         return i40e_dcb_get_num_tc(dcbcfg);
3712 }
3713
3714 /**
3715  * i40e_pf_get_default_tc - Get bitmap for first enabled TC
3716  * @pf: PF being queried
3717  *
3718  * Return a bitmap for first enabled traffic class for this PF.
3719  **/
3720 static u8 i40e_pf_get_default_tc(struct i40e_pf *pf)
3721 {
3722         u8 enabled_tc = pf->hw.func_caps.enabled_tcmap;
3723         u8 i = 0;
3724
3725         if (!enabled_tc)
3726                 return 0x1; /* TC0 */
3727
3728         /* Find the first enabled TC */
3729         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3730                 if (enabled_tc & (1 << i))
3731                         break;
3732         }
3733
3734         return 1 << i;
3735 }
3736
3737 /**
3738  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
3739  * @pf: PF being queried
3740  *
3741  * Return a bitmap for enabled traffic classes for this PF.
3742  **/
3743 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
3744 {
3745         /* If DCB is not enabled for this PF then just return default TC */
3746         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
3747                 return i40e_pf_get_default_tc(pf);
3748
3749         /* MFP mode will have enabled TCs set by FW */
3750         if (pf->flags & I40E_FLAG_MFP_ENABLED)
3751                 return pf->hw.func_caps.enabled_tcmap;
3752
3753         /* SFP mode we want PF to be enabled for all TCs */
3754         return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
3755 }
3756
3757 /**
3758  * i40e_vsi_get_bw_info - Query VSI BW Information
3759  * @vsi: the VSI being queried
3760  *
3761  * Returns 0 on success, negative value on failure
3762  **/
3763 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
3764 {
3765         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
3766         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
3767         struct i40e_pf *pf = vsi->back;
3768         struct i40e_hw *hw = &pf->hw;
3769         i40e_status aq_ret;
3770         u32 tc_bw_max;
3771         int i;
3772
3773         /* Get the VSI level BW configuration */
3774         aq_ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
3775         if (aq_ret) {
3776                 dev_info(&pf->pdev->dev,
3777                          "couldn't get pf vsi bw config, err %d, aq_err %d\n",
3778                          aq_ret, pf->hw.aq.asq_last_status);
3779                 return -EINVAL;
3780         }
3781
3782         /* Get the VSI level BW configuration per TC */
3783         aq_ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
3784                                                   NULL);
3785         if (aq_ret) {
3786                 dev_info(&pf->pdev->dev,
3787                          "couldn't get pf vsi ets bw config, err %d, aq_err %d\n",
3788                          aq_ret, pf->hw.aq.asq_last_status);
3789                 return -EINVAL;
3790         }
3791
3792         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
3793                 dev_info(&pf->pdev->dev,
3794                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
3795                          bw_config.tc_valid_bits,
3796                          bw_ets_config.tc_valid_bits);
3797                 /* Still continuing */
3798         }
3799
3800         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
3801         vsi->bw_max_quanta = bw_config.max_bw;
3802         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
3803                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
3804         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3805                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
3806                 vsi->bw_ets_limit_credits[i] =
3807                                         le16_to_cpu(bw_ets_config.credits[i]);
3808                 /* 3 bits out of 4 for each TC */
3809                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
3810         }
3811
3812         return 0;
3813 }
3814
3815 /**
3816  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
3817  * @vsi: the VSI being configured
3818  * @enabled_tc: TC bitmap
3819  * @bw_credits: BW shared credits per TC
3820  *
3821  * Returns 0 on success, negative value on failure
3822  **/
3823 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
3824                                        u8 *bw_share)
3825 {
3826         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
3827         i40e_status aq_ret;
3828         int i;
3829
3830         bw_data.tc_valid_bits = enabled_tc;
3831         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3832                 bw_data.tc_bw_credits[i] = bw_share[i];
3833
3834         aq_ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
3835                                           NULL);
3836         if (aq_ret) {
3837                 dev_info(&vsi->back->pdev->dev,
3838                          "AQ command Config VSI BW allocation per TC failed = %d\n",
3839                          vsi->back->hw.aq.asq_last_status);
3840                 return -EINVAL;
3841         }
3842
3843         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3844                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
3845
3846         return 0;
3847 }
3848
3849 /**
3850  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
3851  * @vsi: the VSI being configured
3852  * @enabled_tc: TC map to be enabled
3853  *
3854  **/
3855 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
3856 {
3857         struct net_device *netdev = vsi->netdev;
3858         struct i40e_pf *pf = vsi->back;
3859         struct i40e_hw *hw = &pf->hw;
3860         u8 netdev_tc = 0;
3861         int i;
3862         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
3863
3864         if (!netdev)
3865                 return;
3866
3867         if (!enabled_tc) {
3868                 netdev_reset_tc(netdev);
3869                 return;
3870         }
3871
3872         /* Set up actual enabled TCs on the VSI */
3873         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
3874                 return;
3875
3876         /* set per TC queues for the VSI */
3877         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3878                 /* Only set TC queues for enabled tcs
3879                  *
3880                  * e.g. For a VSI that has TC0 and TC3 enabled the
3881                  * enabled_tc bitmap would be 0x00001001; the driver
3882                  * will set the numtc for netdev as 2 that will be
3883                  * referenced by the netdev layer as TC 0 and 1.
3884                  */
3885                 if (vsi->tc_config.enabled_tc & (1 << i))
3886                         netdev_set_tc_queue(netdev,
3887                                         vsi->tc_config.tc_info[i].netdev_tc,
3888                                         vsi->tc_config.tc_info[i].qcount,
3889                                         vsi->tc_config.tc_info[i].qoffset);
3890         }
3891
3892         /* Assign UP2TC map for the VSI */
3893         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
3894                 /* Get the actual TC# for the UP */
3895                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
3896                 /* Get the mapped netdev TC# for the UP */
3897                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
3898                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
3899         }
3900 }
3901
3902 /**
3903  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
3904  * @vsi: the VSI being configured
3905  * @ctxt: the ctxt buffer returned from AQ VSI update param command
3906  **/
3907 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
3908                                       struct i40e_vsi_context *ctxt)
3909 {
3910         /* copy just the sections touched not the entire info
3911          * since not all sections are valid as returned by
3912          * update vsi params
3913          */
3914         vsi->info.mapping_flags = ctxt->info.mapping_flags;
3915         memcpy(&vsi->info.queue_mapping,
3916                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
3917         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
3918                sizeof(vsi->info.tc_mapping));
3919 }
3920
3921 /**
3922  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
3923  * @vsi: VSI to be configured
3924  * @enabled_tc: TC bitmap
3925  *
3926  * This configures a particular VSI for TCs that are mapped to the
3927  * given TC bitmap. It uses default bandwidth share for TCs across
3928  * VSIs to configure TC for a particular VSI.
3929  *
3930  * NOTE:
3931  * It is expected that the VSI queues have been quisced before calling
3932  * this function.
3933  **/
3934 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
3935 {
3936         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
3937         struct i40e_vsi_context ctxt;
3938         int ret = 0;
3939         int i;
3940
3941         /* Check if enabled_tc is same as existing or new TCs */
3942         if (vsi->tc_config.enabled_tc == enabled_tc)
3943                 return ret;
3944
3945         /* Enable ETS TCs with equal BW Share for now across all VSIs */
3946         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3947                 if (enabled_tc & (1 << i))
3948                         bw_share[i] = 1;
3949         }
3950
3951         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
3952         if (ret) {
3953                 dev_info(&vsi->back->pdev->dev,
3954                          "Failed configuring TC map %d for VSI %d\n",
3955                          enabled_tc, vsi->seid);
3956                 goto out;
3957         }
3958
3959         /* Update Queue Pairs Mapping for currently enabled UPs */
3960         ctxt.seid = vsi->seid;
3961         ctxt.pf_num = vsi->back->hw.pf_id;
3962         ctxt.vf_num = 0;
3963         ctxt.uplink_seid = vsi->uplink_seid;
3964         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
3965         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
3966
3967         /* Update the VSI after updating the VSI queue-mapping information */
3968         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3969         if (ret) {
3970                 dev_info(&vsi->back->pdev->dev,
3971                          "update vsi failed, aq_err=%d\n",
3972                          vsi->back->hw.aq.asq_last_status);
3973                 goto out;
3974         }
3975         /* update the local VSI info with updated queue map */
3976         i40e_vsi_update_queue_map(vsi, &ctxt);
3977         vsi->info.valid_sections = 0;
3978
3979         /* Update current VSI BW information */
3980         ret = i40e_vsi_get_bw_info(vsi);
3981         if (ret) {
3982                 dev_info(&vsi->back->pdev->dev,
3983                          "Failed updating vsi bw info, aq_err=%d\n",
3984                          vsi->back->hw.aq.asq_last_status);
3985                 goto out;
3986         }
3987
3988         /* Update the netdev TC setup */
3989         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
3990 out:
3991         return ret;
3992 }
3993
3994 /**
3995  * i40e_veb_config_tc - Configure TCs for given VEB
3996  * @veb: given VEB
3997  * @enabled_tc: TC bitmap
3998  *
3999  * Configures given TC bitmap for VEB (switching) element
4000  **/
4001 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
4002 {
4003         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
4004         struct i40e_pf *pf = veb->pf;
4005         int ret = 0;
4006         int i;
4007
4008         /* No TCs or already enabled TCs just return */
4009         if (!enabled_tc || veb->enabled_tc == enabled_tc)
4010                 return ret;
4011
4012         bw_data.tc_valid_bits = enabled_tc;
4013         /* bw_data.absolute_credits is not set (relative) */
4014
4015         /* Enable ETS TCs with equal BW Share for now */
4016         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4017                 if (enabled_tc & (1 << i))
4018                         bw_data.tc_bw_share_credits[i] = 1;
4019         }
4020
4021         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
4022                                                    &bw_data, NULL);
4023         if (ret) {
4024                 dev_info(&pf->pdev->dev,
4025                          "veb bw config failed, aq_err=%d\n",
4026                          pf->hw.aq.asq_last_status);
4027                 goto out;
4028         }
4029
4030         /* Update the BW information */
4031         ret = i40e_veb_get_bw_info(veb);
4032         if (ret) {
4033                 dev_info(&pf->pdev->dev,
4034                          "Failed getting veb bw config, aq_err=%d\n",
4035                          pf->hw.aq.asq_last_status);
4036         }
4037
4038 out:
4039         return ret;
4040 }
4041
4042 #ifdef CONFIG_I40E_DCB
4043 /**
4044  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
4045  * @pf: PF struct
4046  *
4047  * Reconfigure VEB/VSIs on a given PF; it is assumed that
4048  * the caller would've quiesce all the VSIs before calling
4049  * this function
4050  **/
4051 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
4052 {
4053         u8 tc_map = 0;
4054         int ret;
4055         u8 v;
4056
4057         /* Enable the TCs available on PF to all VEBs */
4058         tc_map = i40e_pf_get_tc_map(pf);
4059         for (v = 0; v < I40E_MAX_VEB; v++) {
4060                 if (!pf->veb[v])
4061                         continue;
4062                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
4063                 if (ret) {
4064                         dev_info(&pf->pdev->dev,
4065                                  "Failed configuring TC for VEB seid=%d\n",
4066                                  pf->veb[v]->seid);
4067                         /* Will try to configure as many components */
4068                 }
4069         }
4070
4071         /* Update each VSI */
4072         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4073                 if (!pf->vsi[v])
4074                         continue;
4075
4076                 /* - Enable all TCs for the LAN VSI
4077                  * - For all others keep them at TC0 for now
4078                  */
4079                 if (v == pf->lan_vsi)
4080                         tc_map = i40e_pf_get_tc_map(pf);
4081                 else
4082                         tc_map = i40e_pf_get_default_tc(pf);
4083
4084                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
4085                 if (ret) {
4086                         dev_info(&pf->pdev->dev,
4087                                  "Failed configuring TC for VSI seid=%d\n",
4088                                  pf->vsi[v]->seid);
4089                         /* Will try to configure as many components */
4090                 } else {
4091                         /* Re-configure VSI vectors based on updated TC map */
4092                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
4093                         if (pf->vsi[v]->netdev)
4094                                 i40e_dcbnl_set_all(pf->vsi[v]);
4095                 }
4096         }
4097 }
4098
4099 /**
4100  * i40e_init_pf_dcb - Initialize DCB configuration
4101  * @pf: PF being configured
4102  *
4103  * Query the current DCB configuration and cache it
4104  * in the hardware structure
4105  **/
4106 static int i40e_init_pf_dcb(struct i40e_pf *pf)
4107 {
4108         struct i40e_hw *hw = &pf->hw;
4109         int err = 0;
4110
4111         if (pf->hw.func_caps.npar_enable)
4112                 goto out;
4113
4114         /* Get the initial DCB configuration */
4115         err = i40e_init_dcb(hw);
4116         if (!err) {
4117                 /* Device/Function is not DCBX capable */
4118                 if ((!hw->func_caps.dcb) ||
4119                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
4120                         dev_info(&pf->pdev->dev,
4121                                  "DCBX offload is not supported or is disabled for this PF.\n");
4122
4123                         if (pf->flags & I40E_FLAG_MFP_ENABLED)
4124                                 goto out;
4125
4126                 } else {
4127                         /* When status is not DISABLED then DCBX in FW */
4128                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
4129                                        DCB_CAP_DCBX_VER_IEEE;
4130                         pf->flags |= I40E_FLAG_DCB_ENABLED;
4131                 }
4132         } else {
4133                 dev_info(&pf->pdev->dev, "AQ Querying DCB configuration failed: %d\n",
4134                          pf->hw.aq.asq_last_status);
4135         }
4136
4137 out:
4138         return err;
4139 }
4140 #endif /* CONFIG_I40E_DCB */
4141 #define SPEED_SIZE 14
4142 #define FC_SIZE 8
4143 /**
4144  * i40e_print_link_message - print link up or down
4145  * @vsi: the VSI for which link needs a message
4146  */
4147 static void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
4148 {
4149         char speed[SPEED_SIZE] = "Unknown";
4150         char fc[FC_SIZE] = "RX/TX";
4151
4152         if (!isup) {
4153                 netdev_info(vsi->netdev, "NIC Link is Down\n");
4154                 return;
4155         }
4156
4157         switch (vsi->back->hw.phy.link_info.link_speed) {
4158         case I40E_LINK_SPEED_40GB:
4159                 strncpy(speed, "40 Gbps", SPEED_SIZE);
4160                 break;
4161         case I40E_LINK_SPEED_10GB:
4162                 strncpy(speed, "10 Gbps", SPEED_SIZE);
4163                 break;
4164         case I40E_LINK_SPEED_1GB:
4165                 strncpy(speed, "1000 Mbps", SPEED_SIZE);
4166                 break;
4167         default:
4168                 break;
4169         }
4170
4171         switch (vsi->back->hw.fc.current_mode) {
4172         case I40E_FC_FULL:
4173                 strncpy(fc, "RX/TX", FC_SIZE);
4174                 break;
4175         case I40E_FC_TX_PAUSE:
4176                 strncpy(fc, "TX", FC_SIZE);
4177                 break;
4178         case I40E_FC_RX_PAUSE:
4179                 strncpy(fc, "RX", FC_SIZE);
4180                 break;
4181         default:
4182                 strncpy(fc, "None", FC_SIZE);
4183                 break;
4184         }
4185
4186         netdev_info(vsi->netdev, "NIC Link is Up %s Full Duplex, Flow Control: %s\n",
4187                     speed, fc);
4188 }
4189
4190 /**
4191  * i40e_up_complete - Finish the last steps of bringing up a connection
4192  * @vsi: the VSI being configured
4193  **/
4194 static int i40e_up_complete(struct i40e_vsi *vsi)
4195 {
4196         struct i40e_pf *pf = vsi->back;
4197         int err;
4198
4199         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4200                 i40e_vsi_configure_msix(vsi);
4201         else
4202                 i40e_configure_msi_and_legacy(vsi);
4203
4204         /* start rings */
4205         err = i40e_vsi_control_rings(vsi, true);
4206         if (err)
4207                 return err;
4208
4209         clear_bit(__I40E_DOWN, &vsi->state);
4210         i40e_napi_enable_all(vsi);
4211         i40e_vsi_enable_irq(vsi);
4212
4213         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
4214             (vsi->netdev)) {
4215                 i40e_print_link_message(vsi, true);
4216                 netif_tx_start_all_queues(vsi->netdev);
4217                 netif_carrier_on(vsi->netdev);
4218         } else if (vsi->netdev) {
4219                 i40e_print_link_message(vsi, false);
4220         }
4221
4222         /* replay FDIR SB filters */
4223         if (vsi->type == I40E_VSI_FDIR)
4224                 i40e_fdir_filter_restore(vsi);
4225         i40e_service_event_schedule(pf);
4226
4227         return 0;
4228 }
4229
4230 /**
4231  * i40e_vsi_reinit_locked - Reset the VSI
4232  * @vsi: the VSI being configured
4233  *
4234  * Rebuild the ring structs after some configuration
4235  * has changed, e.g. MTU size.
4236  **/
4237 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
4238 {
4239         struct i40e_pf *pf = vsi->back;
4240
4241         WARN_ON(in_interrupt());
4242         while (test_and_set_bit(__I40E_CONFIG_BUSY, &pf->state))
4243                 usleep_range(1000, 2000);
4244         i40e_down(vsi);
4245
4246         /* Give a VF some time to respond to the reset.  The
4247          * two second wait is based upon the watchdog cycle in
4248          * the VF driver.
4249          */
4250         if (vsi->type == I40E_VSI_SRIOV)
4251                 msleep(2000);
4252         i40e_up(vsi);
4253         clear_bit(__I40E_CONFIG_BUSY, &pf->state);
4254 }
4255
4256 /**
4257  * i40e_up - Bring the connection back up after being down
4258  * @vsi: the VSI being configured
4259  **/
4260 int i40e_up(struct i40e_vsi *vsi)
4261 {
4262         int err;
4263
4264         err = i40e_vsi_configure(vsi);
4265         if (!err)
4266                 err = i40e_up_complete(vsi);
4267
4268         return err;
4269 }
4270
4271 /**
4272  * i40e_down - Shutdown the connection processing
4273  * @vsi: the VSI being stopped
4274  **/
4275 void i40e_down(struct i40e_vsi *vsi)
4276 {
4277         int i;
4278
4279         /* It is assumed that the caller of this function
4280          * sets the vsi->state __I40E_DOWN bit.
4281          */
4282         if (vsi->netdev) {
4283                 netif_carrier_off(vsi->netdev);
4284                 netif_tx_disable(vsi->netdev);
4285         }
4286         i40e_vsi_disable_irq(vsi);
4287         i40e_vsi_control_rings(vsi, false);
4288         i40e_napi_disable_all(vsi);
4289
4290         for (i = 0; i < vsi->num_queue_pairs; i++) {
4291                 i40e_clean_tx_ring(vsi->tx_rings[i]);
4292                 i40e_clean_rx_ring(vsi->rx_rings[i]);
4293         }
4294 }
4295
4296 /**
4297  * i40e_setup_tc - configure multiple traffic classes
4298  * @netdev: net device to configure
4299  * @tc: number of traffic classes to enable
4300  **/
4301 static int i40e_setup_tc(struct net_device *netdev, u8 tc)
4302 {
4303         struct i40e_netdev_priv *np = netdev_priv(netdev);
4304         struct i40e_vsi *vsi = np->vsi;
4305         struct i40e_pf *pf = vsi->back;
4306         u8 enabled_tc = 0;
4307         int ret = -EINVAL;
4308         int i;
4309
4310         /* Check if DCB enabled to continue */
4311         if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
4312                 netdev_info(netdev, "DCB is not enabled for adapter\n");
4313                 goto exit;
4314         }
4315
4316         /* Check if MFP enabled */
4317         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
4318                 netdev_info(netdev, "Configuring TC not supported in MFP mode\n");
4319                 goto exit;
4320         }
4321
4322         /* Check whether tc count is within enabled limit */
4323         if (tc > i40e_pf_get_num_tc(pf)) {
4324                 netdev_info(netdev, "TC count greater than enabled on link for adapter\n");
4325                 goto exit;
4326         }
4327
4328         /* Generate TC map for number of tc requested */
4329         for (i = 0; i < tc; i++)
4330                 enabled_tc |= (1 << i);
4331
4332         /* Requesting same TC configuration as already enabled */
4333         if (enabled_tc == vsi->tc_config.enabled_tc)
4334                 return 0;
4335
4336         /* Quiesce VSI queues */
4337         i40e_quiesce_vsi(vsi);
4338
4339         /* Configure VSI for enabled TCs */
4340         ret = i40e_vsi_config_tc(vsi, enabled_tc);
4341         if (ret) {
4342                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
4343                             vsi->seid);
4344                 goto exit;
4345         }
4346
4347         /* Unquiesce VSI */
4348         i40e_unquiesce_vsi(vsi);
4349
4350 exit:
4351         return ret;
4352 }
4353
4354 /**
4355  * i40e_open - Called when a network interface is made active
4356  * @netdev: network interface device structure
4357  *
4358  * The open entry point is called when a network interface is made
4359  * active by the system (IFF_UP).  At this point all resources needed
4360  * for transmit and receive operations are allocated, the interrupt
4361  * handler is registered with the OS, the netdev watchdog subtask is
4362  * enabled, and the stack is notified that the interface is ready.
4363  *
4364  * Returns 0 on success, negative value on failure
4365  **/
4366 static int i40e_open(struct net_device *netdev)
4367 {
4368         struct i40e_netdev_priv *np = netdev_priv(netdev);
4369         struct i40e_vsi *vsi = np->vsi;
4370         struct i40e_pf *pf = vsi->back;
4371         int err;
4372
4373         /* disallow open during test or if eeprom is broken */
4374         if (test_bit(__I40E_TESTING, &pf->state) ||
4375             test_bit(__I40E_BAD_EEPROM, &pf->state))
4376                 return -EBUSY;
4377
4378         netif_carrier_off(netdev);
4379
4380         err = i40e_vsi_open(vsi);
4381         if (err)
4382                 return err;
4383
4384         /* configure global TSO hardware offload settings */
4385         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
4386                                                        TCP_FLAG_FIN) >> 16);
4387         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
4388                                                        TCP_FLAG_FIN |
4389                                                        TCP_FLAG_CWR) >> 16);
4390         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
4391
4392 #ifdef CONFIG_I40E_VXLAN
4393         vxlan_get_rx_port(netdev);
4394 #endif
4395
4396         return 0;
4397 }
4398
4399 /**
4400  * i40e_vsi_open -
4401  * @vsi: the VSI to open
4402  *
4403  * Finish initialization of the VSI.
4404  *
4405  * Returns 0 on success, negative value on failure
4406  **/
4407 int i40e_vsi_open(struct i40e_vsi *vsi)
4408 {
4409         struct i40e_pf *pf = vsi->back;
4410         char int_name[IFNAMSIZ];
4411         int err;
4412
4413         /* allocate descriptors */
4414         err = i40e_vsi_setup_tx_resources(vsi);
4415         if (err)
4416                 goto err_setup_tx;
4417         err = i40e_vsi_setup_rx_resources(vsi);
4418         if (err)
4419                 goto err_setup_rx;
4420
4421         err = i40e_vsi_configure(vsi);
4422         if (err)
4423                 goto err_setup_rx;
4424
4425         if (vsi->netdev) {
4426                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
4427                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
4428                 err = i40e_vsi_request_irq(vsi, int_name);
4429                 if (err)
4430                         goto err_setup_rx;
4431
4432                 /* Notify the stack of the actual queue counts. */
4433                 err = netif_set_real_num_tx_queues(vsi->netdev,
4434                                                    vsi->num_queue_pairs);
4435                 if (err)
4436                         goto err_set_queues;
4437
4438                 err = netif_set_real_num_rx_queues(vsi->netdev,
4439                                                    vsi->num_queue_pairs);
4440                 if (err)
4441                         goto err_set_queues;
4442
4443         } else if (vsi->type == I40E_VSI_FDIR) {
4444                 snprintf(int_name, sizeof(int_name) - 1, "%s-fdir",
4445                          dev_driver_string(&pf->pdev->dev));
4446                 err = i40e_vsi_request_irq(vsi, int_name);
4447         } else {
4448                 err = -EINVAL;
4449                 goto err_setup_rx;
4450         }
4451
4452         err = i40e_up_complete(vsi);
4453         if (err)
4454                 goto err_up_complete;
4455
4456         return 0;
4457
4458 err_up_complete:
4459         i40e_down(vsi);
4460 err_set_queues:
4461         i40e_vsi_free_irq(vsi);
4462 err_setup_rx:
4463         i40e_vsi_free_rx_resources(vsi);
4464 err_setup_tx:
4465         i40e_vsi_free_tx_resources(vsi);
4466         if (vsi == pf->vsi[pf->lan_vsi])
4467                 i40e_do_reset(pf, (1 << __I40E_PF_RESET_REQUESTED));
4468
4469         return err;
4470 }
4471
4472 /**
4473  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
4474  * @pf: Pointer to pf
4475  *
4476  * This function destroys the hlist where all the Flow Director
4477  * filters were saved.
4478  **/
4479 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
4480 {
4481         struct i40e_fdir_filter *filter;
4482         struct hlist_node *node2;
4483
4484         hlist_for_each_entry_safe(filter, node2,
4485                                   &pf->fdir_filter_list, fdir_node) {
4486                 hlist_del(&filter->fdir_node);
4487                 kfree(filter);
4488         }
4489         pf->fdir_pf_active_filters = 0;
4490 }
4491
4492 /**
4493  * i40e_close - Disables a network interface
4494  * @netdev: network interface device structure
4495  *
4496  * The close entry point is called when an interface is de-activated
4497  * by the OS.  The hardware is still under the driver's control, but
4498  * this netdev interface is disabled.
4499  *
4500  * Returns 0, this is not allowed to fail
4501  **/
4502 static int i40e_close(struct net_device *netdev)
4503 {
4504         struct i40e_netdev_priv *np = netdev_priv(netdev);
4505         struct i40e_vsi *vsi = np->vsi;
4506
4507         i40e_vsi_close(vsi);
4508
4509         return 0;
4510 }
4511
4512 /**
4513  * i40e_do_reset - Start a PF or Core Reset sequence
4514  * @pf: board private structure
4515  * @reset_flags: which reset is requested
4516  *
4517  * The essential difference in resets is that the PF Reset
4518  * doesn't clear the packet buffers, doesn't reset the PE
4519  * firmware, and doesn't bother the other PFs on the chip.
4520  **/
4521 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags)
4522 {
4523         u32 val;
4524
4525         WARN_ON(in_interrupt());
4526
4527         if (i40e_check_asq_alive(&pf->hw))
4528                 i40e_vc_notify_reset(pf);
4529
4530         /* do the biggest reset indicated */
4531         if (reset_flags & (1 << __I40E_GLOBAL_RESET_REQUESTED)) {
4532
4533                 /* Request a Global Reset
4534                  *
4535                  * This will start the chip's countdown to the actual full
4536                  * chip reset event, and a warning interrupt to be sent
4537                  * to all PFs, including the requestor.  Our handler
4538                  * for the warning interrupt will deal with the shutdown
4539                  * and recovery of the switch setup.
4540                  */
4541                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
4542                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4543                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
4544                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4545
4546         } else if (reset_flags & (1 << __I40E_CORE_RESET_REQUESTED)) {
4547
4548                 /* Request a Core Reset
4549                  *
4550                  * Same as Global Reset, except does *not* include the MAC/PHY
4551                  */
4552                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
4553                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4554                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
4555                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4556                 i40e_flush(&pf->hw);
4557
4558         } else if (reset_flags & (1 << __I40E_EMP_RESET_REQUESTED)) {
4559
4560                 /* Request a Firmware Reset
4561                  *
4562                  * Same as Global reset, plus restarting the
4563                  * embedded firmware engine.
4564                  */
4565                 /* enable EMP Reset */
4566                 val = rd32(&pf->hw, I40E_GLGEN_RSTENA_EMP);
4567                 val |= I40E_GLGEN_RSTENA_EMP_EMP_RST_ENA_MASK;
4568                 wr32(&pf->hw, I40E_GLGEN_RSTENA_EMP, val);
4569
4570                 /* force the reset */
4571                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4572                 val |= I40E_GLGEN_RTRIG_EMPFWR_MASK;
4573                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4574                 i40e_flush(&pf->hw);
4575
4576         } else if (reset_flags & (1 << __I40E_PF_RESET_REQUESTED)) {
4577
4578                 /* Request a PF Reset
4579                  *
4580                  * Resets only the PF-specific registers
4581                  *
4582                  * This goes directly to the tear-down and rebuild of
4583                  * the switch, since we need to do all the recovery as
4584                  * for the Core Reset.
4585                  */
4586                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
4587                 i40e_handle_reset_warning(pf);
4588
4589         } else if (reset_flags & (1 << __I40E_REINIT_REQUESTED)) {
4590                 int v;
4591
4592                 /* Find the VSI(s) that requested a re-init */
4593                 dev_info(&pf->pdev->dev,
4594                          "VSI reinit requested\n");
4595                 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4596                         struct i40e_vsi *vsi = pf->vsi[v];
4597                         if (vsi != NULL &&
4598                             test_bit(__I40E_REINIT_REQUESTED, &vsi->state)) {
4599                                 i40e_vsi_reinit_locked(pf->vsi[v]);
4600                                 clear_bit(__I40E_REINIT_REQUESTED, &vsi->state);
4601                         }
4602                 }
4603
4604                 /* no further action needed, so return now */
4605                 return;
4606         } else {
4607                 dev_info(&pf->pdev->dev,
4608                          "bad reset request 0x%08x\n", reset_flags);
4609                 return;
4610         }
4611 }
4612
4613 #ifdef CONFIG_I40E_DCB
4614 /**
4615  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
4616  * @pf: board private structure
4617  * @old_cfg: current DCB config
4618  * @new_cfg: new DCB config
4619  **/
4620 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
4621                             struct i40e_dcbx_config *old_cfg,
4622                             struct i40e_dcbx_config *new_cfg)
4623 {
4624         bool need_reconfig = false;
4625
4626         /* Check if ETS configuration has changed */
4627         if (memcmp(&new_cfg->etscfg,
4628                    &old_cfg->etscfg,
4629                    sizeof(new_cfg->etscfg))) {
4630                 /* If Priority Table has changed reconfig is needed */
4631                 if (memcmp(&new_cfg->etscfg.prioritytable,
4632                            &old_cfg->etscfg.prioritytable,
4633                            sizeof(new_cfg->etscfg.prioritytable))) {
4634                         need_reconfig = true;
4635                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
4636                 }
4637
4638                 if (memcmp(&new_cfg->etscfg.tcbwtable,
4639                            &old_cfg->etscfg.tcbwtable,
4640                            sizeof(new_cfg->etscfg.tcbwtable)))
4641                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
4642
4643                 if (memcmp(&new_cfg->etscfg.tsatable,
4644                            &old_cfg->etscfg.tsatable,
4645                            sizeof(new_cfg->etscfg.tsatable)))
4646                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
4647         }
4648
4649         /* Check if PFC configuration has changed */
4650         if (memcmp(&new_cfg->pfc,
4651                    &old_cfg->pfc,
4652                    sizeof(new_cfg->pfc))) {
4653                 need_reconfig = true;
4654                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
4655         }
4656
4657         /* Check if APP Table has changed */
4658         if (memcmp(&new_cfg->app,
4659                    &old_cfg->app,
4660                    sizeof(new_cfg->app))) {
4661                 need_reconfig = true;
4662                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
4663         }
4664
4665         return need_reconfig;
4666 }
4667
4668 /**
4669  * i40e_handle_lldp_event - Handle LLDP Change MIB event
4670  * @pf: board private structure
4671  * @e: event info posted on ARQ
4672  **/
4673 static int i40e_handle_lldp_event(struct i40e_pf *pf,
4674                                   struct i40e_arq_event_info *e)
4675 {
4676         struct i40e_aqc_lldp_get_mib *mib =
4677                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
4678         struct i40e_hw *hw = &pf->hw;
4679         struct i40e_dcbx_config *dcbx_cfg = &hw->local_dcbx_config;
4680         struct i40e_dcbx_config tmp_dcbx_cfg;
4681         bool need_reconfig = false;
4682         int ret = 0;
4683         u8 type;
4684
4685         /* Ignore if event is not for Nearest Bridge */
4686         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
4687                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
4688         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
4689                 return ret;
4690
4691         /* Check MIB Type and return if event for Remote MIB update */
4692         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
4693         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
4694                 /* Update the remote cached instance and return */
4695                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
4696                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
4697                                 &hw->remote_dcbx_config);
4698                 goto exit;
4699         }
4700
4701         /* Convert/store the DCBX data from LLDPDU temporarily */
4702         memset(&tmp_dcbx_cfg, 0, sizeof(tmp_dcbx_cfg));
4703         ret = i40e_lldp_to_dcb_config(e->msg_buf, &tmp_dcbx_cfg);
4704         if (ret) {
4705                 /* Error in LLDPDU parsing return */
4706                 dev_info(&pf->pdev->dev, "Failed parsing LLDPDU from event buffer\n");
4707                 goto exit;
4708         }
4709
4710         /* No change detected in DCBX configs */
4711         if (!memcmp(&tmp_dcbx_cfg, dcbx_cfg, sizeof(tmp_dcbx_cfg))) {
4712                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
4713                 goto exit;
4714         }
4715
4716         need_reconfig = i40e_dcb_need_reconfig(pf, dcbx_cfg, &tmp_dcbx_cfg);
4717
4718         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg);
4719
4720         /* Overwrite the new configuration */
4721         *dcbx_cfg = tmp_dcbx_cfg;
4722
4723         if (!need_reconfig)
4724                 goto exit;
4725
4726         /* Reconfiguration needed quiesce all VSIs */
4727         i40e_pf_quiesce_all_vsi(pf);
4728
4729         /* Changes in configuration update VEB/VSI */
4730         i40e_dcb_reconfigure(pf);
4731
4732         i40e_pf_unquiesce_all_vsi(pf);
4733 exit:
4734         return ret;
4735 }
4736 #endif /* CONFIG_I40E_DCB */
4737
4738 /**
4739  * i40e_do_reset_safe - Protected reset path for userland calls.
4740  * @pf: board private structure
4741  * @reset_flags: which reset is requested
4742  *
4743  **/
4744 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
4745 {
4746         rtnl_lock();
4747         i40e_do_reset(pf, reset_flags);
4748         rtnl_unlock();
4749 }
4750
4751 /**
4752  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
4753  * @pf: board private structure
4754  * @e: event info posted on ARQ
4755  *
4756  * Handler for LAN Queue Overflow Event generated by the firmware for PF
4757  * and VF queues
4758  **/
4759 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
4760                                            struct i40e_arq_event_info *e)
4761 {
4762         struct i40e_aqc_lan_overflow *data =
4763                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
4764         u32 queue = le32_to_cpu(data->prtdcb_rupto);
4765         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
4766         struct i40e_hw *hw = &pf->hw;
4767         struct i40e_vf *vf;
4768         u16 vf_id;
4769
4770         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
4771                 queue, qtx_ctl);
4772
4773         /* Queue belongs to VF, find the VF and issue VF reset */
4774         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
4775             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
4776                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
4777                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
4778                 vf_id -= hw->func_caps.vf_base_id;
4779                 vf = &pf->vf[vf_id];
4780                 i40e_vc_notify_vf_reset(vf);
4781                 /* Allow VF to process pending reset notification */
4782                 msleep(20);
4783                 i40e_reset_vf(vf, false);
4784         }
4785 }
4786
4787 /**
4788  * i40e_service_event_complete - Finish up the service event
4789  * @pf: board private structure
4790  **/
4791 static void i40e_service_event_complete(struct i40e_pf *pf)
4792 {
4793         BUG_ON(!test_bit(__I40E_SERVICE_SCHED, &pf->state));
4794
4795         /* flush memory to make sure state is correct before next watchog */
4796         smp_mb__before_clear_bit();
4797         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
4798 }
4799
4800 /**
4801  * i40e_get_current_fd_count - Get the count of FD filters programmed in the HW
4802  * @pf: board private structure
4803  **/
4804 int i40e_get_current_fd_count(struct i40e_pf *pf)
4805 {
4806         int val, fcnt_prog;
4807         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
4808         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
4809                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
4810                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
4811         return fcnt_prog;
4812 }
4813
4814 /**
4815  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
4816  * @pf: board private structure
4817  **/
4818 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
4819 {
4820         u32 fcnt_prog, fcnt_avail;
4821
4822         /* Check if, FD SB or ATR was auto disabled and if there is enough room
4823          * to re-enable
4824          */
4825         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
4826             (pf->flags & I40E_FLAG_FD_SB_ENABLED))
4827                 return;
4828         fcnt_prog = i40e_get_current_fd_count(pf);
4829         fcnt_avail = i40e_get_fd_cnt_all(pf);
4830         if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) {
4831                 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
4832                     (pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED)) {
4833                         pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED;
4834                         dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
4835                 }
4836         }
4837         /* Wait for some more space to be available to turn on ATR */
4838         if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM * 2)) {
4839                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
4840                     (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED)) {
4841                         pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
4842                         dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table now\n");
4843                 }
4844         }
4845 }
4846
4847 /**
4848  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
4849  * @pf: board private structure
4850  **/
4851 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
4852 {
4853         if (!(pf->flags & I40E_FLAG_FDIR_REQUIRES_REINIT))
4854                 return;
4855
4856         /* if interface is down do nothing */
4857         if (test_bit(__I40E_DOWN, &pf->state))
4858                 return;
4859         i40e_fdir_check_and_reenable(pf);
4860
4861         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
4862             (pf->flags & I40E_FLAG_FD_SB_ENABLED))
4863                 pf->flags &= ~I40E_FLAG_FDIR_REQUIRES_REINIT;
4864 }
4865
4866 /**
4867  * i40e_vsi_link_event - notify VSI of a link event
4868  * @vsi: vsi to be notified
4869  * @link_up: link up or down
4870  **/
4871 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
4872 {
4873         if (!vsi)
4874                 return;
4875
4876         switch (vsi->type) {
4877         case I40E_VSI_MAIN:
4878                 if (!vsi->netdev || !vsi->netdev_registered)
4879                         break;
4880
4881                 if (link_up) {
4882                         netif_carrier_on(vsi->netdev);
4883                         netif_tx_wake_all_queues(vsi->netdev);
4884                 } else {
4885                         netif_carrier_off(vsi->netdev);
4886                         netif_tx_stop_all_queues(vsi->netdev);
4887                 }
4888                 break;
4889
4890         case I40E_VSI_SRIOV:
4891                 break;
4892
4893         case I40E_VSI_VMDQ2:
4894         case I40E_VSI_CTRL:
4895         case I40E_VSI_MIRROR:
4896         default:
4897                 /* there is no notification for other VSIs */
4898                 break;
4899         }
4900 }
4901
4902 /**
4903  * i40e_veb_link_event - notify elements on the veb of a link event
4904  * @veb: veb to be notified
4905  * @link_up: link up or down
4906  **/
4907 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
4908 {
4909         struct i40e_pf *pf;
4910         int i;
4911
4912         if (!veb || !veb->pf)
4913                 return;
4914         pf = veb->pf;
4915
4916         /* depth first... */
4917         for (i = 0; i < I40E_MAX_VEB; i++)
4918                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
4919                         i40e_veb_link_event(pf->veb[i], link_up);
4920
4921         /* ... now the local VSIs */
4922         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
4923                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
4924                         i40e_vsi_link_event(pf->vsi[i], link_up);
4925 }
4926
4927 /**
4928  * i40e_link_event - Update netif_carrier status
4929  * @pf: board private structure
4930  **/
4931 static void i40e_link_event(struct i40e_pf *pf)
4932 {
4933         bool new_link, old_link;
4934
4935         new_link = (pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP);
4936         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
4937
4938         if (new_link == old_link)
4939                 return;
4940         if (!test_bit(__I40E_DOWN, &pf->vsi[pf->lan_vsi]->state))
4941                 i40e_print_link_message(pf->vsi[pf->lan_vsi], new_link);
4942
4943         /* Notify the base of the switch tree connected to
4944          * the link.  Floating VEBs are not notified.
4945          */
4946         if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
4947                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
4948         else
4949                 i40e_vsi_link_event(pf->vsi[pf->lan_vsi], new_link);
4950
4951         if (pf->vf)
4952                 i40e_vc_notify_link_state(pf);
4953
4954         if (pf->flags & I40E_FLAG_PTP)
4955                 i40e_ptp_set_increment(pf);
4956 }
4957
4958 /**
4959  * i40e_check_hang_subtask - Check for hung queues and dropped interrupts
4960  * @pf: board private structure
4961  *
4962  * Set the per-queue flags to request a check for stuck queues in the irq
4963  * clean functions, then force interrupts to be sure the irq clean is called.
4964  **/
4965 static void i40e_check_hang_subtask(struct i40e_pf *pf)
4966 {
4967         int i, v;
4968
4969         /* If we're down or resetting, just bail */
4970         if (test_bit(__I40E_CONFIG_BUSY, &pf->state))
4971                 return;
4972
4973         /* for each VSI/netdev
4974          *     for each Tx queue
4975          *         set the check flag
4976          *     for each q_vector
4977          *         force an interrupt
4978          */
4979         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4980                 struct i40e_vsi *vsi = pf->vsi[v];
4981                 int armed = 0;
4982
4983                 if (!pf->vsi[v] ||
4984                     test_bit(__I40E_DOWN, &vsi->state) ||
4985                     (vsi->netdev && !netif_carrier_ok(vsi->netdev)))
4986                         continue;
4987
4988                 for (i = 0; i < vsi->num_queue_pairs; i++) {
4989                         set_check_for_tx_hang(vsi->tx_rings[i]);
4990                         if (test_bit(__I40E_HANG_CHECK_ARMED,
4991                                      &vsi->tx_rings[i]->state))
4992                                 armed++;
4993                 }
4994
4995                 if (armed) {
4996                         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
4997                                 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0,
4998                                      (I40E_PFINT_DYN_CTL0_INTENA_MASK |
4999                                       I40E_PFINT_DYN_CTL0_SWINT_TRIG_MASK));
5000                         } else {
5001                                 u16 vec = vsi->base_vector - 1;
5002                                 u32 val = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
5003                                            I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
5004                                 for (i = 0; i < vsi->num_q_vectors; i++, vec++)
5005                                         wr32(&vsi->back->hw,
5006                                              I40E_PFINT_DYN_CTLN(vec), val);
5007                         }
5008                         i40e_flush(&vsi->back->hw);
5009                 }
5010         }
5011 }
5012
5013 /**
5014  * i40e_watchdog_subtask - Check and bring link up
5015  * @pf: board private structure
5016  **/
5017 static void i40e_watchdog_subtask(struct i40e_pf *pf)
5018 {
5019         int i;
5020
5021         /* if interface is down do nothing */
5022         if (test_bit(__I40E_DOWN, &pf->state) ||
5023             test_bit(__I40E_CONFIG_BUSY, &pf->state))
5024                 return;
5025
5026         /* Update the stats for active netdevs so the network stack
5027          * can look at updated numbers whenever it cares to
5028          */
5029         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
5030                 if (pf->vsi[i] && pf->vsi[i]->netdev)
5031                         i40e_update_stats(pf->vsi[i]);
5032
5033         /* Update the stats for the active switching components */
5034         for (i = 0; i < I40E_MAX_VEB; i++)
5035                 if (pf->veb[i])
5036                         i40e_update_veb_stats(pf->veb[i]);
5037
5038         i40e_ptp_rx_hang(pf->vsi[pf->lan_vsi]);
5039 }
5040
5041 /**
5042  * i40e_reset_subtask - Set up for resetting the device and driver
5043  * @pf: board private structure
5044  **/
5045 static void i40e_reset_subtask(struct i40e_pf *pf)
5046 {
5047         u32 reset_flags = 0;
5048
5049         rtnl_lock();
5050         if (test_bit(__I40E_REINIT_REQUESTED, &pf->state)) {
5051                 reset_flags |= (1 << __I40E_REINIT_REQUESTED);
5052                 clear_bit(__I40E_REINIT_REQUESTED, &pf->state);
5053         }
5054         if (test_bit(__I40E_PF_RESET_REQUESTED, &pf->state)) {
5055                 reset_flags |= (1 << __I40E_PF_RESET_REQUESTED);
5056                 clear_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
5057         }
5058         if (test_bit(__I40E_CORE_RESET_REQUESTED, &pf->state)) {
5059                 reset_flags |= (1 << __I40E_CORE_RESET_REQUESTED);
5060                 clear_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
5061         }
5062         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state)) {
5063                 reset_flags |= (1 << __I40E_GLOBAL_RESET_REQUESTED);
5064                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
5065         }
5066
5067         /* If there's a recovery already waiting, it takes
5068          * precedence before starting a new reset sequence.
5069          */
5070         if (test_bit(__I40E_RESET_INTR_RECEIVED, &pf->state)) {
5071                 i40e_handle_reset_warning(pf);
5072                 goto unlock;
5073         }
5074
5075         /* If we're already down or resetting, just bail */
5076         if (reset_flags &&
5077             !test_bit(__I40E_DOWN, &pf->state) &&
5078             !test_bit(__I40E_CONFIG_BUSY, &pf->state))
5079                 i40e_do_reset(pf, reset_flags);
5080
5081 unlock:
5082         rtnl_unlock();
5083 }
5084
5085 /**
5086  * i40e_handle_link_event - Handle link event
5087  * @pf: board private structure
5088  * @e: event info posted on ARQ
5089  **/
5090 static void i40e_handle_link_event(struct i40e_pf *pf,
5091                                    struct i40e_arq_event_info *e)
5092 {
5093         struct i40e_hw *hw = &pf->hw;
5094         struct i40e_aqc_get_link_status *status =
5095                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
5096         struct i40e_link_status *hw_link_info = &hw->phy.link_info;
5097
5098         /* save off old link status information */
5099         memcpy(&pf->hw.phy.link_info_old, hw_link_info,
5100                sizeof(pf->hw.phy.link_info_old));
5101
5102         /* update link status */
5103         hw_link_info->phy_type = (enum i40e_aq_phy_type)status->phy_type;
5104         hw_link_info->link_speed = (enum i40e_aq_link_speed)status->link_speed;
5105         hw_link_info->link_info = status->link_info;
5106         hw_link_info->an_info = status->an_info;
5107         hw_link_info->ext_info = status->ext_info;
5108         hw_link_info->lse_enable =
5109                 le16_to_cpu(status->command_flags) &
5110                             I40E_AQ_LSE_ENABLE;
5111
5112         /* process the event */
5113         i40e_link_event(pf);
5114
5115         /* Do a new status request to re-enable LSE reporting
5116          * and load new status information into the hw struct,
5117          * then see if the status changed while processing the
5118          * initial event.
5119          */
5120         i40e_aq_get_link_info(&pf->hw, true, NULL, NULL);
5121         i40e_link_event(pf);
5122 }
5123
5124 /**
5125  * i40e_clean_adminq_subtask - Clean the AdminQ rings
5126  * @pf: board private structure
5127  **/
5128 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
5129 {
5130         struct i40e_arq_event_info event;
5131         struct i40e_hw *hw = &pf->hw;
5132         u16 pending, i = 0;
5133         i40e_status ret;
5134         u16 opcode;
5135         u32 val;
5136
5137         if (!test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state))
5138                 return;
5139
5140         event.msg_size = I40E_MAX_AQ_BUF_SIZE;
5141         event.msg_buf = kzalloc(event.msg_size, GFP_KERNEL);
5142         if (!event.msg_buf)
5143                 return;
5144
5145         do {
5146                 event.msg_size = I40E_MAX_AQ_BUF_SIZE; /* reinit each time */
5147                 ret = i40e_clean_arq_element(hw, &event, &pending);
5148                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) {
5149                         dev_info(&pf->pdev->dev, "No ARQ event found\n");
5150                         break;
5151                 } else if (ret) {
5152                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
5153                         break;
5154                 }
5155
5156                 opcode = le16_to_cpu(event.desc.opcode);
5157                 switch (opcode) {
5158
5159                 case i40e_aqc_opc_get_link_status:
5160                         i40e_handle_link_event(pf, &event);
5161                         break;
5162                 case i40e_aqc_opc_send_msg_to_pf:
5163                         ret = i40e_vc_process_vf_msg(pf,
5164                                         le16_to_cpu(event.desc.retval),
5165                                         le32_to_cpu(event.desc.cookie_high),
5166                                         le32_to_cpu(event.desc.cookie_low),
5167                                         event.msg_buf,
5168                                         event.msg_size);
5169                         break;
5170                 case i40e_aqc_opc_lldp_update_mib:
5171                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
5172 #ifdef CONFIG_I40E_DCB
5173                         rtnl_lock();
5174                         ret = i40e_handle_lldp_event(pf, &event);
5175                         rtnl_unlock();
5176 #endif /* CONFIG_I40E_DCB */
5177                         break;
5178                 case i40e_aqc_opc_event_lan_overflow:
5179                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
5180                         i40e_handle_lan_overflow_event(pf, &event);
5181                         break;
5182                 case i40e_aqc_opc_send_msg_to_peer:
5183                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
5184                         break;
5185                 default:
5186                         dev_info(&pf->pdev->dev,
5187                                  "ARQ Error: Unknown event 0x%04x received\n",
5188                                  opcode);
5189                         break;
5190                 }
5191         } while (pending && (i++ < pf->adminq_work_limit));
5192
5193         clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
5194         /* re-enable Admin queue interrupt cause */
5195         val = rd32(hw, I40E_PFINT_ICR0_ENA);
5196         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
5197         wr32(hw, I40E_PFINT_ICR0_ENA, val);
5198         i40e_flush(hw);
5199
5200         kfree(event.msg_buf);
5201 }
5202
5203 /**
5204  * i40e_verify_eeprom - make sure eeprom is good to use
5205  * @pf: board private structure
5206  **/
5207 static void i40e_verify_eeprom(struct i40e_pf *pf)
5208 {
5209         int err;
5210
5211         err = i40e_diag_eeprom_test(&pf->hw);
5212         if (err) {
5213                 /* retry in case of garbage read */
5214                 err = i40e_diag_eeprom_test(&pf->hw);
5215                 if (err) {
5216                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
5217                                  err);
5218                         set_bit(__I40E_BAD_EEPROM, &pf->state);
5219                 }
5220         }
5221
5222         if (!err && test_bit(__I40E_BAD_EEPROM, &pf->state)) {
5223                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
5224                 clear_bit(__I40E_BAD_EEPROM, &pf->state);
5225         }
5226 }
5227
5228 /**
5229  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
5230  * @veb: pointer to the VEB instance
5231  *
5232  * This is a recursive function that first builds the attached VSIs then
5233  * recurses in to build the next layer of VEB.  We track the connections
5234  * through our own index numbers because the seid's from the HW could
5235  * change across the reset.
5236  **/
5237 static int i40e_reconstitute_veb(struct i40e_veb *veb)
5238 {
5239         struct i40e_vsi *ctl_vsi = NULL;
5240         struct i40e_pf *pf = veb->pf;
5241         int v, veb_idx;
5242         int ret;
5243
5244         /* build VSI that owns this VEB, temporarily attached to base VEB */
5245         for (v = 0; v < pf->hw.func_caps.num_vsis && !ctl_vsi; v++) {
5246                 if (pf->vsi[v] &&
5247                     pf->vsi[v]->veb_idx == veb->idx &&
5248                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
5249                         ctl_vsi = pf->vsi[v];
5250                         break;
5251                 }
5252         }
5253         if (!ctl_vsi) {
5254                 dev_info(&pf->pdev->dev,
5255                          "missing owner VSI for veb_idx %d\n", veb->idx);
5256                 ret = -ENOENT;
5257                 goto end_reconstitute;
5258         }
5259         if (ctl_vsi != pf->vsi[pf->lan_vsi])
5260                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
5261         ret = i40e_add_vsi(ctl_vsi);
5262         if (ret) {
5263                 dev_info(&pf->pdev->dev,
5264                          "rebuild of owner VSI failed: %d\n", ret);
5265                 goto end_reconstitute;
5266         }
5267         i40e_vsi_reset_stats(ctl_vsi);
5268
5269         /* create the VEB in the switch and move the VSI onto the VEB */
5270         ret = i40e_add_veb(veb, ctl_vsi);
5271         if (ret)
5272                 goto end_reconstitute;
5273
5274         /* create the remaining VSIs attached to this VEB */
5275         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
5276                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
5277                         continue;
5278
5279                 if (pf->vsi[v]->veb_idx == veb->idx) {
5280                         struct i40e_vsi *vsi = pf->vsi[v];
5281                         vsi->uplink_seid = veb->seid;
5282                         ret = i40e_add_vsi(vsi);
5283                         if (ret) {
5284                                 dev_info(&pf->pdev->dev,
5285                                          "rebuild of vsi_idx %d failed: %d\n",
5286                                          v, ret);
5287                                 goto end_reconstitute;
5288                         }
5289                         i40e_vsi_reset_stats(vsi);
5290                 }
5291         }
5292
5293         /* create any VEBs attached to this VEB - RECURSION */
5294         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
5295                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
5296                         pf->veb[veb_idx]->uplink_seid = veb->seid;
5297                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
5298                         if (ret)
5299                                 break;
5300                 }
5301         }
5302
5303 end_reconstitute:
5304         return ret;
5305 }
5306
5307 /**
5308  * i40e_get_capabilities - get info about the HW
5309  * @pf: the PF struct
5310  **/
5311 static int i40e_get_capabilities(struct i40e_pf *pf)
5312 {
5313         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
5314         u16 data_size;
5315         int buf_len;
5316         int err;
5317
5318         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
5319         do {
5320                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
5321                 if (!cap_buf)
5322                         return -ENOMEM;
5323
5324                 /* this loads the data into the hw struct for us */
5325                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
5326                                             &data_size,
5327                                             i40e_aqc_opc_list_func_capabilities,
5328                                             NULL);
5329                 /* data loaded, buffer no longer needed */
5330                 kfree(cap_buf);
5331
5332                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
5333                         /* retry with a larger buffer */
5334                         buf_len = data_size;
5335                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
5336                         dev_info(&pf->pdev->dev,
5337                                  "capability discovery failed: aq=%d\n",
5338                                  pf->hw.aq.asq_last_status);
5339                         return -ENODEV;
5340                 }
5341         } while (err);
5342
5343         if (((pf->hw.aq.fw_maj_ver == 2) && (pf->hw.aq.fw_min_ver < 22)) ||
5344             (pf->hw.aq.fw_maj_ver < 2)) {
5345                 pf->hw.func_caps.num_msix_vectors++;
5346                 pf->hw.func_caps.num_msix_vectors_vf++;
5347         }
5348
5349         if (pf->hw.debug_mask & I40E_DEBUG_USER)
5350                 dev_info(&pf->pdev->dev,
5351                          "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
5352                          pf->hw.pf_id, pf->hw.func_caps.num_vfs,
5353                          pf->hw.func_caps.num_msix_vectors,
5354                          pf->hw.func_caps.num_msix_vectors_vf,
5355                          pf->hw.func_caps.fd_filters_guaranteed,
5356                          pf->hw.func_caps.fd_filters_best_effort,
5357                          pf->hw.func_caps.num_tx_qp,
5358                          pf->hw.func_caps.num_vsis);
5359
5360 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
5361                        + pf->hw.func_caps.num_vfs)
5362         if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) {
5363                 dev_info(&pf->pdev->dev,
5364                          "got num_vsis %d, setting num_vsis to %d\n",
5365                          pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
5366                 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
5367         }
5368
5369         return 0;
5370 }
5371
5372 static int i40e_vsi_clear(struct i40e_vsi *vsi);
5373
5374 /**
5375  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
5376  * @pf: board private structure
5377  **/
5378 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
5379 {
5380         struct i40e_vsi *vsi;
5381         int i;
5382
5383         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
5384                 return;
5385
5386         /* find existing VSI and see if it needs configuring */
5387         vsi = NULL;
5388         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
5389                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
5390                         vsi = pf->vsi[i];
5391                         break;
5392                 }
5393         }
5394
5395         /* create a new VSI if none exists */
5396         if (!vsi) {
5397                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
5398                                      pf->vsi[pf->lan_vsi]->seid, 0);
5399                 if (!vsi) {
5400                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
5401                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
5402                         return;
5403                 }
5404         }
5405
5406         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
5407 }
5408
5409 /**
5410  * i40e_fdir_teardown - release the Flow Director resources
5411  * @pf: board private structure
5412  **/
5413 static void i40e_fdir_teardown(struct i40e_pf *pf)
5414 {
5415         int i;
5416
5417         i40e_fdir_filter_exit(pf);
5418         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
5419                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
5420                         i40e_vsi_release(pf->vsi[i]);
5421                         break;
5422                 }
5423         }
5424 }
5425
5426 /**
5427  * i40e_prep_for_reset - prep for the core to reset
5428  * @pf: board private structure
5429  *
5430  * Close up the VFs and other things in prep for pf Reset.
5431   **/
5432 static int i40e_prep_for_reset(struct i40e_pf *pf)
5433 {
5434         struct i40e_hw *hw = &pf->hw;
5435         i40e_status ret = 0;
5436         u32 v;
5437
5438         clear_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
5439         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
5440                 return 0;
5441
5442         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
5443
5444         /* quiesce the VSIs and their queues that are not already DOWN */
5445         i40e_pf_quiesce_all_vsi(pf);
5446
5447         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
5448                 if (pf->vsi[v])
5449                         pf->vsi[v]->seid = 0;
5450         }
5451
5452         i40e_shutdown_adminq(&pf->hw);
5453
5454         /* call shutdown HMC */
5455         if (hw->hmc.hmc_obj) {
5456                 ret = i40e_shutdown_lan_hmc(hw);
5457                 if (ret) {
5458                         dev_warn(&pf->pdev->dev,
5459                                  "shutdown_lan_hmc failed: %d\n", ret);
5460                         clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
5461                 }
5462         }
5463         return ret;
5464 }
5465
5466 /**
5467  * i40e_send_version - update firmware with driver version
5468  * @pf: PF struct
5469  */
5470 static void i40e_send_version(struct i40e_pf *pf)
5471 {
5472         struct i40e_driver_version dv;
5473
5474         dv.major_version = DRV_VERSION_MAJOR;
5475         dv.minor_version = DRV_VERSION_MINOR;
5476         dv.build_version = DRV_VERSION_BUILD;
5477         dv.subbuild_version = 0;
5478         strncpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
5479         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
5480 }
5481
5482 /**
5483  * i40e_reset_and_rebuild - reset and rebuild using a saved config
5484  * @pf: board private structure
5485  * @reinit: if the Main VSI needs to re-initialized.
5486  **/
5487 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit)
5488 {
5489         struct i40e_hw *hw = &pf->hw;
5490         i40e_status ret;
5491         u32 v;
5492
5493         /* Now we wait for GRST to settle out.
5494          * We don't have to delete the VEBs or VSIs from the hw switch
5495          * because the reset will make them disappear.
5496          */
5497         ret = i40e_pf_reset(hw);
5498         if (ret) {
5499                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
5500                 goto end_core_reset;
5501         }
5502         pf->pfr_count++;
5503
5504         if (test_bit(__I40E_DOWN, &pf->state))
5505                 goto end_core_reset;
5506         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
5507
5508         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
5509         ret = i40e_init_adminq(&pf->hw);
5510         if (ret) {
5511                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, %d\n", ret);
5512                 goto end_core_reset;
5513         }
5514
5515         /* re-verify the eeprom if we just had an EMP reset */
5516         if (test_bit(__I40E_EMP_RESET_REQUESTED, &pf->state)) {
5517                 clear_bit(__I40E_EMP_RESET_REQUESTED, &pf->state);
5518                 i40e_verify_eeprom(pf);
5519         }
5520
5521         i40e_clear_pxe_mode(hw);
5522         ret = i40e_get_capabilities(pf);
5523         if (ret) {
5524                 dev_info(&pf->pdev->dev, "i40e_get_capabilities failed, %d\n",
5525                          ret);
5526                 goto end_core_reset;
5527         }
5528
5529         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
5530                                 hw->func_caps.num_rx_qp,
5531                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
5532         if (ret) {
5533                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
5534                 goto end_core_reset;
5535         }
5536         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
5537         if (ret) {
5538                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
5539                 goto end_core_reset;
5540         }
5541
5542 #ifdef CONFIG_I40E_DCB
5543         ret = i40e_init_pf_dcb(pf);
5544         if (ret) {
5545                 dev_info(&pf->pdev->dev, "init_pf_dcb failed: %d\n", ret);
5546                 goto end_core_reset;
5547         }
5548 #endif /* CONFIG_I40E_DCB */
5549
5550         /* do basic switch setup */
5551         ret = i40e_setup_pf_switch(pf, reinit);
5552         if (ret)
5553                 goto end_core_reset;
5554
5555         /* Rebuild the VSIs and VEBs that existed before reset.
5556          * They are still in our local switch element arrays, so only
5557          * need to rebuild the switch model in the HW.
5558          *
5559          * If there were VEBs but the reconstitution failed, we'll try
5560          * try to recover minimal use by getting the basic PF VSI working.
5561          */
5562         if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) {
5563                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
5564                 /* find the one VEB connected to the MAC, and find orphans */
5565                 for (v = 0; v < I40E_MAX_VEB; v++) {
5566                         if (!pf->veb[v])
5567                                 continue;
5568
5569                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
5570                             pf->veb[v]->uplink_seid == 0) {
5571                                 ret = i40e_reconstitute_veb(pf->veb[v]);
5572
5573                                 if (!ret)
5574                                         continue;
5575
5576                                 /* If Main VEB failed, we're in deep doodoo,
5577                                  * so give up rebuilding the switch and set up
5578                                  * for minimal rebuild of PF VSI.
5579                                  * If orphan failed, we'll report the error
5580                                  * but try to keep going.
5581                                  */
5582                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
5583                                         dev_info(&pf->pdev->dev,
5584                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
5585                                                  ret);
5586                                         pf->vsi[pf->lan_vsi]->uplink_seid
5587                                                                 = pf->mac_seid;
5588                                         break;
5589                                 } else if (pf->veb[v]->uplink_seid == 0) {
5590                                         dev_info(&pf->pdev->dev,
5591                                                  "rebuild of orphan VEB failed: %d\n",
5592                                                  ret);
5593                                 }
5594                         }
5595                 }
5596         }
5597
5598         if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) {
5599                 dev_info(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
5600                 /* no VEB, so rebuild only the Main VSI */
5601                 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]);
5602                 if (ret) {
5603                         dev_info(&pf->pdev->dev,
5604                                  "rebuild of Main VSI failed: %d\n", ret);
5605                         goto end_core_reset;
5606                 }
5607         }
5608
5609         /* reinit the misc interrupt */
5610         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5611                 ret = i40e_setup_misc_vector(pf);
5612
5613         /* restart the VSIs that were rebuilt and running before the reset */
5614         i40e_pf_unquiesce_all_vsi(pf);
5615
5616         if (pf->num_alloc_vfs) {
5617                 for (v = 0; v < pf->num_alloc_vfs; v++)
5618                         i40e_reset_vf(&pf->vf[v], true);
5619         }
5620
5621         /* tell the firmware that we're starting */
5622         i40e_send_version(pf);
5623
5624 end_core_reset:
5625         clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
5626 }
5627
5628 /**
5629  * i40e_handle_reset_warning - prep for the pf to reset, reset and rebuild
5630  * @pf: board private structure
5631  *
5632  * Close up the VFs and other things in prep for a Core Reset,
5633  * then get ready to rebuild the world.
5634  **/
5635 static void i40e_handle_reset_warning(struct i40e_pf *pf)
5636 {
5637         i40e_status ret;
5638
5639         ret = i40e_prep_for_reset(pf);
5640         if (!ret)
5641                 i40e_reset_and_rebuild(pf, false);
5642 }
5643
5644 /**
5645  * i40e_handle_mdd_event
5646  * @pf: pointer to the pf structure
5647  *
5648  * Called from the MDD irq handler to identify possibly malicious vfs
5649  **/
5650 static void i40e_handle_mdd_event(struct i40e_pf *pf)
5651 {
5652         struct i40e_hw *hw = &pf->hw;
5653         bool mdd_detected = false;
5654         struct i40e_vf *vf;
5655         u32 reg;
5656         int i;
5657
5658         if (!test_bit(__I40E_MDD_EVENT_PENDING, &pf->state))
5659                 return;
5660
5661         /* find what triggered the MDD event */
5662         reg = rd32(hw, I40E_GL_MDET_TX);
5663         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
5664                 u8 func = (reg & I40E_GL_MDET_TX_FUNCTION_MASK)
5665                                 >> I40E_GL_MDET_TX_FUNCTION_SHIFT;
5666                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_SHIFT)
5667                                 >> I40E_GL_MDET_TX_EVENT_SHIFT;
5668                 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK)
5669                                 >> I40E_GL_MDET_TX_QUEUE_SHIFT;
5670                 dev_info(&pf->pdev->dev,
5671                          "Malicious Driver Detection event 0x%02x on TX queue %d of function 0x%02x\n",
5672                          event, queue, func);
5673                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
5674                 mdd_detected = true;
5675         }
5676         reg = rd32(hw, I40E_GL_MDET_RX);
5677         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
5678                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK)
5679                                 >> I40E_GL_MDET_RX_FUNCTION_SHIFT;
5680                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_SHIFT)
5681                                 >> I40E_GL_MDET_RX_EVENT_SHIFT;
5682                 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK)
5683                                 >> I40E_GL_MDET_RX_QUEUE_SHIFT;
5684                 dev_info(&pf->pdev->dev,
5685                          "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
5686                          event, queue, func);
5687                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
5688                 mdd_detected = true;
5689         }
5690
5691         /* see if one of the VFs needs its hand slapped */
5692         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
5693                 vf = &(pf->vf[i]);
5694                 reg = rd32(hw, I40E_VP_MDET_TX(i));
5695                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
5696                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
5697                         vf->num_mdd_events++;
5698                         dev_info(&pf->pdev->dev, "MDD TX event on VF %d\n", i);
5699                 }
5700
5701                 reg = rd32(hw, I40E_VP_MDET_RX(i));
5702                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
5703                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
5704                         vf->num_mdd_events++;
5705                         dev_info(&pf->pdev->dev, "MDD RX event on VF %d\n", i);
5706                 }
5707
5708                 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
5709                         dev_info(&pf->pdev->dev,
5710                                  "Too many MDD events on VF %d, disabled\n", i);
5711                         dev_info(&pf->pdev->dev,
5712                                  "Use PF Control I/F to re-enable the VF\n");
5713                         set_bit(I40E_VF_STAT_DISABLED, &vf->vf_states);
5714                 }
5715         }
5716
5717         /* re-enable mdd interrupt cause */
5718         clear_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
5719         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
5720         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
5721         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
5722         i40e_flush(hw);
5723 }
5724
5725 #ifdef CONFIG_I40E_VXLAN
5726 /**
5727  * i40e_sync_vxlan_filters_subtask - Sync the VSI filter list with HW
5728  * @pf: board private structure
5729  **/
5730 static void i40e_sync_vxlan_filters_subtask(struct i40e_pf *pf)
5731 {
5732         struct i40e_hw *hw = &pf->hw;
5733         i40e_status ret;
5734         u8 filter_index;
5735         __be16 port;
5736         int i;
5737
5738         if (!(pf->flags & I40E_FLAG_VXLAN_FILTER_SYNC))
5739                 return;
5740
5741         pf->flags &= ~I40E_FLAG_VXLAN_FILTER_SYNC;
5742
5743         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
5744                 if (pf->pending_vxlan_bitmap & (1 << i)) {
5745                         pf->pending_vxlan_bitmap &= ~(1 << i);
5746                         port = pf->vxlan_ports[i];
5747                         ret = port ?
5748                               i40e_aq_add_udp_tunnel(hw, ntohs(port),
5749                                                      I40E_AQC_TUNNEL_TYPE_VXLAN,
5750                                                      &filter_index, NULL)
5751                               : i40e_aq_del_udp_tunnel(hw, i, NULL);
5752
5753                         if (ret) {
5754                                 dev_info(&pf->pdev->dev, "Failed to execute AQ command for %s port %d with index %d\n",
5755                                          port ? "adding" : "deleting",
5756                                          ntohs(port), port ? i : i);
5757
5758                                 pf->vxlan_ports[i] = 0;
5759                         } else {
5760                                 dev_info(&pf->pdev->dev, "%s port %d with AQ command with index %d\n",
5761                                          port ? "Added" : "Deleted",
5762                                          ntohs(port), port ? i : filter_index);
5763                         }
5764                 }
5765         }
5766 }
5767
5768 #endif
5769 /**
5770  * i40e_service_task - Run the driver's async subtasks
5771  * @work: pointer to work_struct containing our data
5772  **/
5773 static void i40e_service_task(struct work_struct *work)
5774 {
5775         struct i40e_pf *pf = container_of(work,
5776                                           struct i40e_pf,
5777                                           service_task);
5778         unsigned long start_time = jiffies;
5779
5780         i40e_reset_subtask(pf);
5781         i40e_handle_mdd_event(pf);
5782         i40e_vc_process_vflr_event(pf);
5783         i40e_watchdog_subtask(pf);
5784         i40e_fdir_reinit_subtask(pf);
5785         i40e_check_hang_subtask(pf);
5786         i40e_sync_filters_subtask(pf);
5787 #ifdef CONFIG_I40E_VXLAN
5788         i40e_sync_vxlan_filters_subtask(pf);
5789 #endif
5790         i40e_clean_adminq_subtask(pf);
5791
5792         i40e_service_event_complete(pf);
5793
5794         /* If the tasks have taken longer than one timer cycle or there
5795          * is more work to be done, reschedule the service task now
5796          * rather than wait for the timer to tick again.
5797          */
5798         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
5799             test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state)            ||
5800             test_bit(__I40E_MDD_EVENT_PENDING, &pf->state)               ||
5801             test_bit(__I40E_VFLR_EVENT_PENDING, &pf->state))
5802                 i40e_service_event_schedule(pf);
5803 }
5804
5805 /**
5806  * i40e_service_timer - timer callback
5807  * @data: pointer to PF struct
5808  **/
5809 static void i40e_service_timer(unsigned long data)
5810 {
5811         struct i40e_pf *pf = (struct i40e_pf *)data;
5812
5813         mod_timer(&pf->service_timer,
5814                   round_jiffies(jiffies + pf->service_timer_period));
5815         i40e_service_event_schedule(pf);
5816 }
5817
5818 /**
5819  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
5820  * @vsi: the VSI being configured
5821  **/
5822 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
5823 {
5824         struct i40e_pf *pf = vsi->back;
5825
5826         switch (vsi->type) {
5827         case I40E_VSI_MAIN:
5828                 vsi->alloc_queue_pairs = pf->num_lan_qps;
5829                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
5830                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5831                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5832                         vsi->num_q_vectors = pf->num_lan_msix;
5833                 else
5834                         vsi->num_q_vectors = 1;
5835
5836                 break;
5837
5838         case I40E_VSI_FDIR:
5839                 vsi->alloc_queue_pairs = 1;
5840                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
5841                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5842                 vsi->num_q_vectors = 1;
5843                 break;
5844
5845         case I40E_VSI_VMDQ2:
5846                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
5847                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
5848                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5849                 vsi->num_q_vectors = pf->num_vmdq_msix;
5850                 break;
5851
5852         case I40E_VSI_SRIOV:
5853                 vsi->alloc_queue_pairs = pf->num_vf_qps;
5854                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
5855                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5856                 break;
5857
5858         default:
5859                 WARN_ON(1);
5860                 return -ENODATA;
5861         }
5862
5863         return 0;
5864 }
5865
5866 /**
5867  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
5868  * @type: VSI pointer
5869  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
5870  *
5871  * On error: returns error code (negative)
5872  * On success: returns 0
5873  **/
5874 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
5875 {
5876         int size;
5877         int ret = 0;
5878
5879         /* allocate memory for both Tx and Rx ring pointers */
5880         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2;
5881         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
5882         if (!vsi->tx_rings)
5883                 return -ENOMEM;
5884         vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs];
5885
5886         if (alloc_qvectors) {
5887                 /* allocate memory for q_vector pointers */
5888                 size = sizeof(struct i40e_q_vectors *) * vsi->num_q_vectors;
5889                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
5890                 if (!vsi->q_vectors) {
5891                         ret = -ENOMEM;
5892                         goto err_vectors;
5893                 }
5894         }
5895         return ret;
5896
5897 err_vectors:
5898         kfree(vsi->tx_rings);
5899         return ret;
5900 }
5901
5902 /**
5903  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
5904  * @pf: board private structure
5905  * @type: type of VSI
5906  *
5907  * On error: returns error code (negative)
5908  * On success: returns vsi index in PF (positive)
5909  **/
5910 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
5911 {
5912         int ret = -ENODEV;
5913         struct i40e_vsi *vsi;
5914         int vsi_idx;
5915         int i;
5916
5917         /* Need to protect the allocation of the VSIs at the PF level */
5918         mutex_lock(&pf->switch_mutex);
5919
5920         /* VSI list may be fragmented if VSI creation/destruction has
5921          * been happening.  We can afford to do a quick scan to look
5922          * for any free VSIs in the list.
5923          *
5924          * find next empty vsi slot, looping back around if necessary
5925          */
5926         i = pf->next_vsi;
5927         while (i < pf->hw.func_caps.num_vsis && pf->vsi[i])
5928                 i++;
5929         if (i >= pf->hw.func_caps.num_vsis) {
5930                 i = 0;
5931                 while (i < pf->next_vsi && pf->vsi[i])
5932                         i++;
5933         }
5934
5935         if (i < pf->hw.func_caps.num_vsis && !pf->vsi[i]) {
5936                 vsi_idx = i;             /* Found one! */
5937         } else {
5938                 ret = -ENODEV;
5939                 goto unlock_pf;  /* out of VSI slots! */
5940         }
5941         pf->next_vsi = ++i;
5942
5943         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
5944         if (!vsi) {
5945                 ret = -ENOMEM;
5946                 goto unlock_pf;
5947         }
5948         vsi->type = type;
5949         vsi->back = pf;
5950         set_bit(__I40E_DOWN, &vsi->state);
5951         vsi->flags = 0;
5952         vsi->idx = vsi_idx;
5953         vsi->rx_itr_setting = pf->rx_itr_default;
5954         vsi->tx_itr_setting = pf->tx_itr_default;
5955         vsi->netdev_registered = false;
5956         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
5957         INIT_LIST_HEAD(&vsi->mac_filter_list);
5958         vsi->irqs_ready = false;
5959
5960         ret = i40e_set_num_rings_in_vsi(vsi);
5961         if (ret)
5962                 goto err_rings;
5963
5964         ret = i40e_vsi_alloc_arrays(vsi, true);
5965         if (ret)
5966                 goto err_rings;
5967
5968         /* Setup default MSIX irq handler for VSI */
5969         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
5970
5971         pf->vsi[vsi_idx] = vsi;
5972         ret = vsi_idx;
5973         goto unlock_pf;
5974
5975 err_rings:
5976         pf->next_vsi = i - 1;
5977         kfree(vsi);
5978 unlock_pf:
5979         mutex_unlock(&pf->switch_mutex);
5980         return ret;
5981 }
5982
5983 /**
5984  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
5985  * @type: VSI pointer
5986  * @free_qvectors: a bool to specify if q_vectors need to be freed.
5987  *
5988  * On error: returns error code (negative)
5989  * On success: returns 0
5990  **/
5991 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
5992 {
5993         /* free the ring and vector containers */
5994         if (free_qvectors) {
5995                 kfree(vsi->q_vectors);
5996                 vsi->q_vectors = NULL;
5997         }
5998         kfree(vsi->tx_rings);
5999         vsi->tx_rings = NULL;
6000         vsi->rx_rings = NULL;
6001 }
6002
6003 /**
6004  * i40e_vsi_clear - Deallocate the VSI provided
6005  * @vsi: the VSI being un-configured
6006  **/
6007 static int i40e_vsi_clear(struct i40e_vsi *vsi)
6008 {
6009         struct i40e_pf *pf;
6010
6011         if (!vsi)
6012                 return 0;
6013
6014         if (!vsi->back)
6015                 goto free_vsi;
6016         pf = vsi->back;
6017
6018         mutex_lock(&pf->switch_mutex);
6019         if (!pf->vsi[vsi->idx]) {
6020                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
6021                         vsi->idx, vsi->idx, vsi, vsi->type);
6022                 goto unlock_vsi;
6023         }
6024
6025         if (pf->vsi[vsi->idx] != vsi) {
6026                 dev_err(&pf->pdev->dev,
6027                         "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
6028                         pf->vsi[vsi->idx]->idx,
6029                         pf->vsi[vsi->idx],
6030                         pf->vsi[vsi->idx]->type,
6031                         vsi->idx, vsi, vsi->type);
6032                 goto unlock_vsi;
6033         }
6034
6035         /* updates the pf for this cleared vsi */
6036         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
6037         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
6038
6039         i40e_vsi_free_arrays(vsi, true);
6040
6041         pf->vsi[vsi->idx] = NULL;
6042         if (vsi->idx < pf->next_vsi)
6043                 pf->next_vsi = vsi->idx;
6044
6045 unlock_vsi:
6046         mutex_unlock(&pf->switch_mutex);
6047 free_vsi:
6048         kfree(vsi);
6049
6050         return 0;
6051 }
6052
6053 /**
6054  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
6055  * @vsi: the VSI being cleaned
6056  **/
6057 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
6058 {
6059         int i;
6060
6061         if (vsi->tx_rings && vsi->tx_rings[0]) {
6062                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
6063                         kfree_rcu(vsi->tx_rings[i], rcu);
6064                         vsi->tx_rings[i] = NULL;
6065                         vsi->rx_rings[i] = NULL;
6066                 }
6067         }
6068 }
6069
6070 /**
6071  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
6072  * @vsi: the VSI being configured
6073  **/
6074 static int i40e_alloc_rings(struct i40e_vsi *vsi)
6075 {
6076         struct i40e_ring *tx_ring, *rx_ring;
6077         struct i40e_pf *pf = vsi->back;
6078         int i;
6079
6080         /* Set basic values in the rings to be used later during open() */
6081         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
6082                 /* allocate space for both Tx and Rx in one shot */
6083                 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL);
6084                 if (!tx_ring)
6085                         goto err_out;
6086
6087                 tx_ring->queue_index = i;
6088                 tx_ring->reg_idx = vsi->base_queue + i;
6089                 tx_ring->ring_active = false;
6090                 tx_ring->vsi = vsi;
6091                 tx_ring->netdev = vsi->netdev;
6092                 tx_ring->dev = &pf->pdev->dev;
6093                 tx_ring->count = vsi->num_desc;
6094                 tx_ring->size = 0;
6095                 tx_ring->dcb_tc = 0;
6096                 vsi->tx_rings[i] = tx_ring;
6097
6098                 rx_ring = &tx_ring[1];
6099                 rx_ring->queue_index = i;
6100                 rx_ring->reg_idx = vsi->base_queue + i;
6101                 rx_ring->ring_active = false;
6102                 rx_ring->vsi = vsi;
6103                 rx_ring->netdev = vsi->netdev;
6104                 rx_ring->dev = &pf->pdev->dev;
6105                 rx_ring->count = vsi->num_desc;
6106                 rx_ring->size = 0;
6107                 rx_ring->dcb_tc = 0;
6108                 if (pf->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED)
6109                         set_ring_16byte_desc_enabled(rx_ring);
6110                 else
6111                         clear_ring_16byte_desc_enabled(rx_ring);
6112                 vsi->rx_rings[i] = rx_ring;
6113         }
6114
6115         return 0;
6116
6117 err_out:
6118         i40e_vsi_clear_rings(vsi);
6119         return -ENOMEM;
6120 }
6121
6122 /**
6123  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
6124  * @pf: board private structure
6125  * @vectors: the number of MSI-X vectors to request
6126  *
6127  * Returns the number of vectors reserved, or error
6128  **/
6129 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
6130 {
6131         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
6132                                         I40E_MIN_MSIX, vectors);
6133         if (vectors < 0) {
6134                 dev_info(&pf->pdev->dev,
6135                          "MSI-X vector reservation failed: %d\n", vectors);
6136                 vectors = 0;
6137         }
6138
6139         return vectors;
6140 }
6141
6142 /**
6143  * i40e_init_msix - Setup the MSIX capability
6144  * @pf: board private structure
6145  *
6146  * Work with the OS to set up the MSIX vectors needed.
6147  *
6148  * Returns 0 on success, negative on failure
6149  **/
6150 static int i40e_init_msix(struct i40e_pf *pf)
6151 {
6152         i40e_status err = 0;
6153         struct i40e_hw *hw = &pf->hw;
6154         int v_budget, i;
6155         int vec;
6156
6157         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
6158                 return -ENODEV;
6159
6160         /* The number of vectors we'll request will be comprised of:
6161          *   - Add 1 for "other" cause for Admin Queue events, etc.
6162          *   - The number of LAN queue pairs
6163          *      - Queues being used for RSS.
6164          *              We don't need as many as max_rss_size vectors.
6165          *              use rss_size instead in the calculation since that
6166          *              is governed by number of cpus in the system.
6167          *      - assumes symmetric Tx/Rx pairing
6168          *   - The number of VMDq pairs
6169          * Once we count this up, try the request.
6170          *
6171          * If we can't get what we want, we'll simplify to nearly nothing
6172          * and try again.  If that still fails, we punt.
6173          */
6174         pf->num_lan_msix = pf->num_lan_qps - (pf->rss_size_max - pf->rss_size);
6175         pf->num_vmdq_msix = pf->num_vmdq_qps;
6176         v_budget = 1 + pf->num_lan_msix;
6177         v_budget += (pf->num_vmdq_vsis * pf->num_vmdq_msix);
6178         if (pf->flags & I40E_FLAG_FD_SB_ENABLED)
6179                 v_budget++;
6180
6181         /* Scale down if necessary, and the rings will share vectors */
6182         v_budget = min_t(int, v_budget, hw->func_caps.num_msix_vectors);
6183
6184         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
6185                                    GFP_KERNEL);
6186         if (!pf->msix_entries)
6187                 return -ENOMEM;
6188
6189         for (i = 0; i < v_budget; i++)
6190                 pf->msix_entries[i].entry = i;
6191         vec = i40e_reserve_msix_vectors(pf, v_budget);
6192         if (vec < I40E_MIN_MSIX) {
6193                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
6194                 kfree(pf->msix_entries);
6195                 pf->msix_entries = NULL;
6196                 return -ENODEV;
6197
6198         } else if (vec == I40E_MIN_MSIX) {
6199                 /* Adjust for minimal MSIX use */
6200                 dev_info(&pf->pdev->dev, "Features disabled, not enough MSI-X vectors\n");
6201                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
6202                 pf->num_vmdq_vsis = 0;
6203                 pf->num_vmdq_qps = 0;
6204                 pf->num_vmdq_msix = 0;
6205                 pf->num_lan_qps = 1;
6206                 pf->num_lan_msix = 1;
6207
6208         } else if (vec != v_budget) {
6209                 /* Scale vector usage down */
6210                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
6211                 vec--;                    /* reserve the misc vector */
6212
6213                 /* partition out the remaining vectors */
6214                 switch (vec) {
6215                 case 2:
6216                         pf->num_vmdq_vsis = 1;
6217                         pf->num_lan_msix = 1;
6218                         break;
6219                 case 3:
6220                         pf->num_vmdq_vsis = 1;
6221                         pf->num_lan_msix = 2;
6222                         break;
6223                 default:
6224                         pf->num_lan_msix = min_t(int, (vec / 2),
6225                                                  pf->num_lan_qps);
6226                         pf->num_vmdq_vsis = min_t(int, (vec - pf->num_lan_msix),
6227                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
6228                         break;
6229                 }
6230         }
6231
6232         return err;
6233 }
6234
6235 /**
6236  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
6237  * @vsi: the VSI being configured
6238  * @v_idx: index of the vector in the vsi struct
6239  *
6240  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
6241  **/
6242 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
6243 {
6244         struct i40e_q_vector *q_vector;
6245
6246         /* allocate q_vector */
6247         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
6248         if (!q_vector)
6249                 return -ENOMEM;
6250
6251         q_vector->vsi = vsi;
6252         q_vector->v_idx = v_idx;
6253         cpumask_set_cpu(v_idx, &q_vector->affinity_mask);
6254         if (vsi->netdev)
6255                 netif_napi_add(vsi->netdev, &q_vector->napi,
6256                                i40e_napi_poll, NAPI_POLL_WEIGHT);
6257
6258         q_vector->rx.latency_range = I40E_LOW_LATENCY;
6259         q_vector->tx.latency_range = I40E_LOW_LATENCY;
6260
6261         /* tie q_vector and vsi together */
6262         vsi->q_vectors[v_idx] = q_vector;
6263
6264         return 0;
6265 }
6266
6267 /**
6268  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
6269  * @vsi: the VSI being configured
6270  *
6271  * We allocate one q_vector per queue interrupt.  If allocation fails we
6272  * return -ENOMEM.
6273  **/
6274 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
6275 {
6276         struct i40e_pf *pf = vsi->back;
6277         int v_idx, num_q_vectors;
6278         int err;
6279
6280         /* if not MSIX, give the one vector only to the LAN VSI */
6281         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
6282                 num_q_vectors = vsi->num_q_vectors;
6283         else if (vsi == pf->vsi[pf->lan_vsi])
6284                 num_q_vectors = 1;
6285         else
6286                 return -EINVAL;
6287
6288         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
6289                 err = i40e_vsi_alloc_q_vector(vsi, v_idx);
6290                 if (err)
6291                         goto err_out;
6292         }
6293
6294         return 0;
6295
6296 err_out:
6297         while (v_idx--)
6298                 i40e_free_q_vector(vsi, v_idx);
6299
6300         return err;
6301 }
6302
6303 /**
6304  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
6305  * @pf: board private structure to initialize
6306  **/
6307 static void i40e_init_interrupt_scheme(struct i40e_pf *pf)
6308 {
6309         int err = 0;
6310
6311         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
6312                 err = i40e_init_msix(pf);
6313                 if (err) {
6314                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
6315                                        I40E_FLAG_RSS_ENABLED    |
6316                                        I40E_FLAG_DCB_ENABLED    |
6317                                        I40E_FLAG_SRIOV_ENABLED  |
6318                                        I40E_FLAG_FD_SB_ENABLED  |
6319                                        I40E_FLAG_FD_ATR_ENABLED |
6320                                        I40E_FLAG_VMDQ_ENABLED);
6321
6322                         /* rework the queue expectations without MSIX */
6323                         i40e_determine_queue_usage(pf);
6324                 }
6325         }
6326
6327         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
6328             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
6329                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
6330                 err = pci_enable_msi(pf->pdev);
6331                 if (err) {
6332                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n", err);
6333                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
6334                 }
6335         }
6336
6337         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
6338                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
6339
6340         /* track first vector for misc interrupts */
6341         err = i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT-1);
6342 }
6343
6344 /**
6345  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
6346  * @pf: board private structure
6347  *
6348  * This sets up the handler for MSIX 0, which is used to manage the
6349  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
6350  * when in MSI or Legacy interrupt mode.
6351  **/
6352 static int i40e_setup_misc_vector(struct i40e_pf *pf)
6353 {
6354         struct i40e_hw *hw = &pf->hw;
6355         int err = 0;
6356
6357         /* Only request the irq if this is the first time through, and
6358          * not when we're rebuilding after a Reset
6359          */
6360         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
6361                 err = request_irq(pf->msix_entries[0].vector,
6362                                   i40e_intr, 0, pf->misc_int_name, pf);
6363                 if (err) {
6364                         dev_info(&pf->pdev->dev,
6365                                  "request_irq for %s failed: %d\n",
6366                                  pf->misc_int_name, err);
6367                         return -EFAULT;
6368                 }
6369         }
6370
6371         i40e_enable_misc_int_causes(hw);
6372
6373         /* associate no queues to the misc vector */
6374         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
6375         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
6376
6377         i40e_flush(hw);
6378
6379         i40e_irq_dynamic_enable_icr0(pf);
6380
6381         return err;
6382 }
6383
6384 /**
6385  * i40e_config_rss - Prepare for RSS if used
6386  * @pf: board private structure
6387  **/
6388 static int i40e_config_rss(struct i40e_pf *pf)
6389 {
6390         /* Set of random keys generated using kernel random number generator */
6391         static const u32 seed[I40E_PFQF_HKEY_MAX_INDEX + 1] = {0x41b01687,
6392                                 0x183cfd8c, 0xce880440, 0x580cbc3c, 0x35897377,
6393                                 0x328b25e1, 0x4fa98922, 0xb7d90c14, 0xd5bad70d,
6394                                 0xcd15a2c1, 0xe8580225, 0x4a1e9d11, 0xfe5731be};
6395         struct i40e_hw *hw = &pf->hw;
6396         u32 lut = 0;
6397         int i, j;
6398         u64 hena;
6399
6400         /* Fill out hash function seed */
6401         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
6402                 wr32(hw, I40E_PFQF_HKEY(i), seed[i]);
6403
6404         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
6405         hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) |
6406                 ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32);
6407         hena |= I40E_DEFAULT_RSS_HENA;
6408         wr32(hw, I40E_PFQF_HENA(0), (u32)hena);
6409         wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
6410
6411         /* Populate the LUT with max no. of queues in round robin fashion */
6412         for (i = 0, j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
6413
6414                 /* The assumption is that lan qp count will be the highest
6415                  * qp count for any PF VSI that needs RSS.
6416                  * If multiple VSIs need RSS support, all the qp counts
6417                  * for those VSIs should be a power of 2 for RSS to work.
6418                  * If LAN VSI is the only consumer for RSS then this requirement
6419                  * is not necessary.
6420                  */
6421                 if (j == pf->rss_size)
6422                         j = 0;
6423                 /* lut = 4-byte sliding window of 4 lut entries */
6424                 lut = (lut << 8) | (j &
6425                          ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
6426                 /* On i = 3, we have 4 entries in lut; write to the register */
6427                 if ((i & 3) == 3)
6428                         wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
6429         }
6430         i40e_flush(hw);
6431
6432         return 0;
6433 }
6434
6435 /**
6436  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
6437  * @pf: board private structure
6438  * @queue_count: the requested queue count for rss.
6439  *
6440  * returns 0 if rss is not enabled, if enabled returns the final rss queue
6441  * count which may be different from the requested queue count.
6442  **/
6443 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
6444 {
6445         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
6446                 return 0;
6447
6448         queue_count = min_t(int, queue_count, pf->rss_size_max);
6449         queue_count = rounddown_pow_of_two(queue_count);
6450
6451         if (queue_count != pf->rss_size) {
6452                 i40e_prep_for_reset(pf);
6453
6454                 pf->rss_size = queue_count;
6455
6456                 i40e_reset_and_rebuild(pf, true);
6457                 i40e_config_rss(pf);
6458         }
6459         dev_info(&pf->pdev->dev, "RSS count:  %d\n", pf->rss_size);
6460         return pf->rss_size;
6461 }
6462
6463 /**
6464  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
6465  * @pf: board private structure to initialize
6466  *
6467  * i40e_sw_init initializes the Adapter private data structure.
6468  * Fields are initialized based on PCI device information and
6469  * OS network device settings (MTU size).
6470  **/
6471 static int i40e_sw_init(struct i40e_pf *pf)
6472 {
6473         int err = 0;
6474         int size;
6475
6476         pf->msg_enable = netif_msg_init(I40E_DEFAULT_MSG_ENABLE,
6477                                 (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK));
6478         pf->hw.debug_mask = pf->msg_enable | I40E_DEBUG_DIAG;
6479         if (debug != -1 && debug != I40E_DEFAULT_MSG_ENABLE) {
6480                 if (I40E_DEBUG_USER & debug)
6481                         pf->hw.debug_mask = debug;
6482                 pf->msg_enable = netif_msg_init((debug & ~I40E_DEBUG_USER),
6483                                                 I40E_DEFAULT_MSG_ENABLE);
6484         }
6485
6486         /* Set default capability flags */
6487         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
6488                     I40E_FLAG_MSI_ENABLED     |
6489                     I40E_FLAG_MSIX_ENABLED    |
6490                     I40E_FLAG_RX_1BUF_ENABLED;
6491
6492         /* Set default ITR */
6493         pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF;
6494         pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF;
6495
6496         /* Depending on PF configurations, it is possible that the RSS
6497          * maximum might end up larger than the available queues
6498          */
6499         pf->rss_size_max = 0x1 << pf->hw.func_caps.rss_table_entry_width;
6500         pf->rss_size_max = min_t(int, pf->rss_size_max,
6501                                  pf->hw.func_caps.num_tx_qp);
6502         if (pf->hw.func_caps.rss) {
6503                 pf->flags |= I40E_FLAG_RSS_ENABLED;
6504                 pf->rss_size = min_t(int, pf->rss_size_max, num_online_cpus());
6505                 pf->rss_size = rounddown_pow_of_two(pf->rss_size);
6506         } else {
6507                 pf->rss_size = 1;
6508         }
6509
6510         /* MFP mode enabled */
6511         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.mfp_mode_1) {
6512                 pf->flags |= I40E_FLAG_MFP_ENABLED;
6513                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
6514         }
6515
6516         /* FW/NVM is not yet fixed in this regard */
6517         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
6518             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
6519                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
6520                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
6521                 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) {
6522                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
6523                 } else {
6524                         dev_info(&pf->pdev->dev,
6525                                  "Flow Director Sideband mode Disabled in MFP mode\n");
6526                 }
6527                 pf->fdir_pf_filter_count =
6528                                  pf->hw.func_caps.fd_filters_guaranteed;
6529                 pf->hw.fdir_shared_filter_count =
6530                                  pf->hw.func_caps.fd_filters_best_effort;
6531         }
6532
6533         if (pf->hw.func_caps.vmdq) {
6534                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
6535                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
6536                 pf->num_vmdq_qps = I40E_DEFAULT_QUEUES_PER_VMDQ;
6537         }
6538
6539 #ifdef CONFIG_PCI_IOV
6540         if (pf->hw.func_caps.num_vfs) {
6541                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
6542                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
6543                 pf->num_req_vfs = min_t(int,
6544                                         pf->hw.func_caps.num_vfs,
6545                                         I40E_MAX_VF_COUNT);
6546         }
6547 #endif /* CONFIG_PCI_IOV */
6548         pf->eeprom_version = 0xDEAD;
6549         pf->lan_veb = I40E_NO_VEB;
6550         pf->lan_vsi = I40E_NO_VSI;
6551
6552         /* set up queue assignment tracking */
6553         size = sizeof(struct i40e_lump_tracking)
6554                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
6555         pf->qp_pile = kzalloc(size, GFP_KERNEL);
6556         if (!pf->qp_pile) {
6557                 err = -ENOMEM;
6558                 goto sw_init_done;
6559         }
6560         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
6561         pf->qp_pile->search_hint = 0;
6562
6563         /* set up vector assignment tracking */
6564         size = sizeof(struct i40e_lump_tracking)
6565                 + (sizeof(u16) * pf->hw.func_caps.num_msix_vectors);
6566         pf->irq_pile = kzalloc(size, GFP_KERNEL);
6567         if (!pf->irq_pile) {
6568                 kfree(pf->qp_pile);
6569                 err = -ENOMEM;
6570                 goto sw_init_done;
6571         }
6572         pf->irq_pile->num_entries = pf->hw.func_caps.num_msix_vectors;
6573         pf->irq_pile->search_hint = 0;
6574
6575         mutex_init(&pf->switch_mutex);
6576
6577 sw_init_done:
6578         return err;
6579 }
6580
6581 /**
6582  * i40e_set_ntuple - set the ntuple feature flag and take action
6583  * @pf: board private structure to initialize
6584  * @features: the feature set that the stack is suggesting
6585  *
6586  * returns a bool to indicate if reset needs to happen
6587  **/
6588 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
6589 {
6590         bool need_reset = false;
6591
6592         /* Check if Flow Director n-tuple support was enabled or disabled.  If
6593          * the state changed, we need to reset.
6594          */
6595         if (features & NETIF_F_NTUPLE) {
6596                 /* Enable filters and mark for reset */
6597                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
6598                         need_reset = true;
6599                 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
6600         } else {
6601                 /* turn off filters, mark for reset and clear SW filter list */
6602                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
6603                         need_reset = true;
6604                         i40e_fdir_filter_exit(pf);
6605                 }
6606                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
6607                 /* if ATR was disabled it can be re-enabled. */
6608                 if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED))
6609                         pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
6610         }
6611         return need_reset;
6612 }
6613
6614 /**
6615  * i40e_set_features - set the netdev feature flags
6616  * @netdev: ptr to the netdev being adjusted
6617  * @features: the feature set that the stack is suggesting
6618  **/
6619 static int i40e_set_features(struct net_device *netdev,
6620                              netdev_features_t features)
6621 {
6622         struct i40e_netdev_priv *np = netdev_priv(netdev);
6623         struct i40e_vsi *vsi = np->vsi;
6624         struct i40e_pf *pf = vsi->back;
6625         bool need_reset;
6626
6627         if (features & NETIF_F_HW_VLAN_CTAG_RX)
6628                 i40e_vlan_stripping_enable(vsi);
6629         else
6630                 i40e_vlan_stripping_disable(vsi);
6631
6632         need_reset = i40e_set_ntuple(pf, features);
6633
6634         if (need_reset)
6635                 i40e_do_reset(pf, (1 << __I40E_PF_RESET_REQUESTED));
6636
6637         return 0;
6638 }
6639
6640 #ifdef CONFIG_I40E_VXLAN
6641 /**
6642  * i40e_get_vxlan_port_idx - Lookup a possibly offloaded for Rx UDP port
6643  * @pf: board private structure
6644  * @port: The UDP port to look up
6645  *
6646  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
6647  **/
6648 static u8 i40e_get_vxlan_port_idx(struct i40e_pf *pf, __be16 port)
6649 {
6650         u8 i;
6651
6652         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
6653                 if (pf->vxlan_ports[i] == port)
6654                         return i;
6655         }
6656
6657         return i;
6658 }
6659
6660 /**
6661  * i40e_add_vxlan_port - Get notifications about VXLAN ports that come up
6662  * @netdev: This physical port's netdev
6663  * @sa_family: Socket Family that VXLAN is notifying us about
6664  * @port: New UDP port number that VXLAN started listening to
6665  **/
6666 static void i40e_add_vxlan_port(struct net_device *netdev,
6667                                 sa_family_t sa_family, __be16 port)
6668 {
6669         struct i40e_netdev_priv *np = netdev_priv(netdev);
6670         struct i40e_vsi *vsi = np->vsi;
6671         struct i40e_pf *pf = vsi->back;
6672         u8 next_idx;
6673         u8 idx;
6674
6675         if (sa_family == AF_INET6)
6676                 return;
6677
6678         idx = i40e_get_vxlan_port_idx(pf, port);
6679
6680         /* Check if port already exists */
6681         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
6682                 netdev_info(netdev, "Port %d already offloaded\n", ntohs(port));
6683                 return;
6684         }
6685
6686         /* Now check if there is space to add the new port */
6687         next_idx = i40e_get_vxlan_port_idx(pf, 0);
6688
6689         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
6690                 netdev_info(netdev, "Maximum number of UDP ports reached, not adding port %d\n",
6691                             ntohs(port));
6692                 return;
6693         }
6694
6695         /* New port: add it and mark its index in the bitmap */
6696         pf->vxlan_ports[next_idx] = port;
6697         pf->pending_vxlan_bitmap |= (1 << next_idx);
6698
6699         pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC;
6700 }
6701
6702 /**
6703  * i40e_del_vxlan_port - Get notifications about VXLAN ports that go away
6704  * @netdev: This physical port's netdev
6705  * @sa_family: Socket Family that VXLAN is notifying us about
6706  * @port: UDP port number that VXLAN stopped listening to
6707  **/
6708 static void i40e_del_vxlan_port(struct net_device *netdev,
6709                                 sa_family_t sa_family, __be16 port)
6710 {
6711         struct i40e_netdev_priv *np = netdev_priv(netdev);
6712         struct i40e_vsi *vsi = np->vsi;
6713         struct i40e_pf *pf = vsi->back;
6714         u8 idx;
6715
6716         if (sa_family == AF_INET6)
6717                 return;
6718
6719         idx = i40e_get_vxlan_port_idx(pf, port);
6720
6721         /* Check if port already exists */
6722         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
6723                 /* if port exists, set it to 0 (mark for deletion)
6724                  * and make it pending
6725                  */
6726                 pf->vxlan_ports[idx] = 0;
6727
6728                 pf->pending_vxlan_bitmap |= (1 << idx);
6729
6730                 pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC;
6731         } else {
6732                 netdev_warn(netdev, "Port %d was not found, not deleting\n",
6733                             ntohs(port));
6734         }
6735 }
6736
6737 #endif
6738 #ifdef HAVE_FDB_OPS
6739 #ifdef USE_CONST_DEV_UC_CHAR
6740 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
6741                             struct net_device *dev,
6742                             const unsigned char *addr,
6743                             u16 flags)
6744 #else
6745 static int i40e_ndo_fdb_add(struct ndmsg *ndm,
6746                             struct net_device *dev,
6747                             unsigned char *addr,
6748                             u16 flags)
6749 #endif
6750 {
6751         struct i40e_netdev_priv *np = netdev_priv(dev);
6752         struct i40e_pf *pf = np->vsi->back;
6753         int err = 0;
6754
6755         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
6756                 return -EOPNOTSUPP;
6757
6758         /* Hardware does not support aging addresses so if a
6759          * ndm_state is given only allow permanent addresses
6760          */
6761         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
6762                 netdev_info(dev, "FDB only supports static addresses\n");
6763                 return -EINVAL;
6764         }
6765
6766         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
6767                 err = dev_uc_add_excl(dev, addr);
6768         else if (is_multicast_ether_addr(addr))
6769                 err = dev_mc_add_excl(dev, addr);
6770         else
6771                 err = -EINVAL;
6772
6773         /* Only return duplicate errors if NLM_F_EXCL is set */
6774         if (err == -EEXIST && !(flags & NLM_F_EXCL))
6775                 err = 0;
6776
6777         return err;
6778 }
6779
6780 #ifndef USE_DEFAULT_FDB_DEL_DUMP
6781 #ifdef USE_CONST_DEV_UC_CHAR
6782 static int i40e_ndo_fdb_del(struct ndmsg *ndm,
6783                             struct net_device *dev,
6784                             const unsigned char *addr)
6785 #else
6786 static int i40e_ndo_fdb_del(struct ndmsg *ndm,
6787                             struct net_device *dev,
6788                             unsigned char *addr)
6789 #endif
6790 {
6791         struct i40e_netdev_priv *np = netdev_priv(dev);
6792         struct i40e_pf *pf = np->vsi->back;
6793         int err = -EOPNOTSUPP;
6794
6795         if (ndm->ndm_state & NUD_PERMANENT) {
6796                 netdev_info(dev, "FDB only supports static addresses\n");
6797                 return -EINVAL;
6798         }
6799
6800         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
6801                 if (is_unicast_ether_addr(addr))
6802                         err = dev_uc_del(dev, addr);
6803                 else if (is_multicast_ether_addr(addr))
6804                         err = dev_mc_del(dev, addr);
6805                 else
6806                         err = -EINVAL;
6807         }
6808
6809         return err;
6810 }
6811
6812 static int i40e_ndo_fdb_dump(struct sk_buff *skb,
6813                              struct netlink_callback *cb,
6814                              struct net_device *dev,
6815                              int idx)
6816 {
6817         struct i40e_netdev_priv *np = netdev_priv(dev);
6818         struct i40e_pf *pf = np->vsi->back;
6819
6820         if (pf->flags & I40E_FLAG_SRIOV_ENABLED)
6821                 idx = ndo_dflt_fdb_dump(skb, cb, dev, idx);
6822
6823         return idx;
6824 }
6825
6826 #endif /* USE_DEFAULT_FDB_DEL_DUMP */
6827 #endif /* HAVE_FDB_OPS */
6828 static const struct net_device_ops i40e_netdev_ops = {
6829         .ndo_open               = i40e_open,
6830         .ndo_stop               = i40e_close,
6831         .ndo_start_xmit         = i40e_lan_xmit_frame,
6832         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
6833         .ndo_set_rx_mode        = i40e_set_rx_mode,
6834         .ndo_validate_addr      = eth_validate_addr,
6835         .ndo_set_mac_address    = i40e_set_mac,
6836         .ndo_change_mtu         = i40e_change_mtu,
6837         .ndo_do_ioctl           = i40e_ioctl,
6838         .ndo_tx_timeout         = i40e_tx_timeout,
6839         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
6840         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
6841 #ifdef CONFIG_NET_POLL_CONTROLLER
6842         .ndo_poll_controller    = i40e_netpoll,
6843 #endif
6844         .ndo_setup_tc           = i40e_setup_tc,
6845         .ndo_set_features       = i40e_set_features,
6846         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
6847         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
6848         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
6849         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
6850         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
6851 #ifdef CONFIG_I40E_VXLAN
6852         .ndo_add_vxlan_port     = i40e_add_vxlan_port,
6853         .ndo_del_vxlan_port     = i40e_del_vxlan_port,
6854 #endif
6855 #ifdef HAVE_FDB_OPS
6856         .ndo_fdb_add            = i40e_ndo_fdb_add,
6857 #ifndef USE_DEFAULT_FDB_DEL_DUMP
6858         .ndo_fdb_del            = i40e_ndo_fdb_del,
6859         .ndo_fdb_dump           = i40e_ndo_fdb_dump,
6860 #endif
6861 #endif
6862 };
6863
6864 /**
6865  * i40e_config_netdev - Setup the netdev flags
6866  * @vsi: the VSI being configured
6867  *
6868  * Returns 0 on success, negative value on failure
6869  **/
6870 static int i40e_config_netdev(struct i40e_vsi *vsi)
6871 {
6872         u8 brdcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
6873         struct i40e_pf *pf = vsi->back;
6874         struct i40e_hw *hw = &pf->hw;
6875         struct i40e_netdev_priv *np;
6876         struct net_device *netdev;
6877         u8 mac_addr[ETH_ALEN];
6878         int etherdev_size;
6879
6880         etherdev_size = sizeof(struct i40e_netdev_priv);
6881         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
6882         if (!netdev)
6883                 return -ENOMEM;
6884
6885         vsi->netdev = netdev;
6886         np = netdev_priv(netdev);
6887         np->vsi = vsi;
6888
6889         netdev->hw_enc_features |= NETIF_F_IP_CSUM       |
6890                                   NETIF_F_GSO_UDP_TUNNEL |
6891                                   NETIF_F_TSO;
6892
6893         netdev->features = NETIF_F_SG                  |
6894                            NETIF_F_IP_CSUM             |
6895                            NETIF_F_SCTP_CSUM           |
6896                            NETIF_F_HIGHDMA             |
6897                            NETIF_F_GSO_UDP_TUNNEL      |
6898                            NETIF_F_HW_VLAN_CTAG_TX     |
6899                            NETIF_F_HW_VLAN_CTAG_RX     |
6900                            NETIF_F_HW_VLAN_CTAG_FILTER |
6901                            NETIF_F_IPV6_CSUM           |
6902                            NETIF_F_TSO                 |
6903                            NETIF_F_TSO_ECN             |
6904                            NETIF_F_TSO6                |
6905                            NETIF_F_RXCSUM              |
6906                            NETIF_F_RXHASH              |
6907                            0;
6908
6909         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
6910                 netdev->features |= NETIF_F_NTUPLE;
6911
6912         /* copy netdev features into list of user selectable features */
6913         netdev->hw_features |= netdev->features;
6914
6915         if (vsi->type == I40E_VSI_MAIN) {
6916                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
6917                 memcpy(mac_addr, hw->mac.perm_addr, ETH_ALEN);
6918         } else {
6919                 /* relate the VSI_VMDQ name to the VSI_MAIN name */
6920                 snprintf(netdev->name, IFNAMSIZ, "%sv%%d",
6921                          pf->vsi[pf->lan_vsi]->netdev->name);
6922                 random_ether_addr(mac_addr);
6923                 i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY, false, false);
6924         }
6925         i40e_add_filter(vsi, brdcast, I40E_VLAN_ANY, false, false);
6926
6927         memcpy(netdev->dev_addr, mac_addr, ETH_ALEN);
6928         memcpy(netdev->perm_addr, mac_addr, ETH_ALEN);
6929         /* vlan gets same features (except vlan offload)
6930          * after any tweaks for specific VSI types
6931          */
6932         netdev->vlan_features = netdev->features & ~(NETIF_F_HW_VLAN_CTAG_TX |
6933                                                      NETIF_F_HW_VLAN_CTAG_RX |
6934                                                    NETIF_F_HW_VLAN_CTAG_FILTER);
6935         netdev->priv_flags |= IFF_UNICAST_FLT;
6936         netdev->priv_flags |= IFF_SUPP_NOFCS;
6937         /* Setup netdev TC information */
6938         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
6939
6940         netdev->netdev_ops = &i40e_netdev_ops;
6941         netdev->watchdog_timeo = 5 * HZ;
6942         i40e_set_ethtool_ops(netdev);
6943
6944         return 0;
6945 }
6946
6947 /**
6948  * i40e_vsi_delete - Delete a VSI from the switch
6949  * @vsi: the VSI being removed
6950  *
6951  * Returns 0 on success, negative value on failure
6952  **/
6953 static void i40e_vsi_delete(struct i40e_vsi *vsi)
6954 {
6955         /* remove default VSI is not allowed */
6956         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
6957                 return;
6958
6959         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
6960 }
6961
6962 /**
6963  * i40e_add_vsi - Add a VSI to the switch
6964  * @vsi: the VSI being configured
6965  *
6966  * This initializes a VSI context depending on the VSI type to be added and
6967  * passes it down to the add_vsi aq command.
6968  **/
6969 static int i40e_add_vsi(struct i40e_vsi *vsi)
6970 {
6971         int ret = -ENODEV;
6972         struct i40e_mac_filter *f, *ftmp;
6973         struct i40e_pf *pf = vsi->back;
6974         struct i40e_hw *hw = &pf->hw;
6975         struct i40e_vsi_context ctxt;
6976         u8 enabled_tc = 0x1; /* TC0 enabled */
6977         int f_count = 0;
6978
6979         memset(&ctxt, 0, sizeof(ctxt));
6980         switch (vsi->type) {
6981         case I40E_VSI_MAIN:
6982                 /* The PF's main VSI is already setup as part of the
6983                  * device initialization, so we'll not bother with
6984                  * the add_vsi call, but we will retrieve the current
6985                  * VSI context.
6986                  */
6987                 ctxt.seid = pf->main_vsi_seid;
6988                 ctxt.pf_num = pf->hw.pf_id;
6989                 ctxt.vf_num = 0;
6990                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6991                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6992                 if (ret) {
6993                         dev_info(&pf->pdev->dev,
6994                                  "couldn't get pf vsi config, err %d, aq_err %d\n",
6995                                  ret, pf->hw.aq.asq_last_status);
6996                         return -ENOENT;
6997                 }
6998                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
6999                 vsi->info.valid_sections = 0;
7000
7001                 vsi->seid = ctxt.seid;
7002                 vsi->id = ctxt.vsi_number;
7003
7004                 enabled_tc = i40e_pf_get_tc_map(pf);
7005
7006                 /* MFP mode setup queue map and update VSI */
7007                 if (pf->flags & I40E_FLAG_MFP_ENABLED) {
7008                         memset(&ctxt, 0, sizeof(ctxt));
7009                         ctxt.seid = pf->main_vsi_seid;
7010                         ctxt.pf_num = pf->hw.pf_id;
7011                         ctxt.vf_num = 0;
7012                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
7013                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7014                         if (ret) {
7015                                 dev_info(&pf->pdev->dev,
7016                                          "update vsi failed, aq_err=%d\n",
7017                                          pf->hw.aq.asq_last_status);
7018                                 ret = -ENOENT;
7019                                 goto err;
7020                         }
7021                         /* update the local VSI info queue map */
7022                         i40e_vsi_update_queue_map(vsi, &ctxt);
7023                         vsi->info.valid_sections = 0;
7024                 } else {
7025                         /* Default/Main VSI is only enabled for TC0
7026                          * reconfigure it to enable all TCs that are
7027                          * available on the port in SFP mode.
7028                          */
7029                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
7030                         if (ret) {
7031                                 dev_info(&pf->pdev->dev,
7032                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %d, aq_err %d\n",
7033                                          enabled_tc, ret,
7034                                          pf->hw.aq.asq_last_status);
7035                                 ret = -ENOENT;
7036                         }
7037                 }
7038                 break;
7039
7040         case I40E_VSI_FDIR:
7041                 ctxt.pf_num = hw->pf_id;
7042                 ctxt.vf_num = 0;
7043                 ctxt.uplink_seid = vsi->uplink_seid;
7044                 ctxt.connection_type = 0x1;     /* regular data port */
7045                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
7046                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
7047                 break;
7048
7049         case I40E_VSI_VMDQ2:
7050                 ctxt.pf_num = hw->pf_id;
7051                 ctxt.vf_num = 0;
7052                 ctxt.uplink_seid = vsi->uplink_seid;
7053                 ctxt.connection_type = 0x1;     /* regular data port */
7054                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
7055
7056                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
7057
7058                 /* This VSI is connected to VEB so the switch_id
7059                  * should be set to zero by default.
7060                  */
7061                 ctxt.info.switch_id = 0;
7062                 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
7063                 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
7064
7065                 /* Setup the VSI tx/rx queue map for TC0 only for now */
7066                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
7067                 break;
7068
7069         case I40E_VSI_SRIOV:
7070                 ctxt.pf_num = hw->pf_id;
7071                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
7072                 ctxt.uplink_seid = vsi->uplink_seid;
7073                 ctxt.connection_type = 0x1;     /* regular data port */
7074                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
7075
7076                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
7077
7078                 /* This VSI is connected to VEB so the switch_id
7079                  * should be set to zero by default.
7080                  */
7081                 ctxt.info.switch_id = cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
7082
7083                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
7084                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
7085                 /* Setup the VSI tx/rx queue map for TC0 only for now */
7086                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
7087                 break;
7088
7089         default:
7090                 return -ENODEV;
7091         }
7092
7093         if (vsi->type != I40E_VSI_MAIN) {
7094                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
7095                 if (ret) {
7096                         dev_info(&vsi->back->pdev->dev,
7097                                  "add vsi failed, aq_err=%d\n",
7098                                  vsi->back->hw.aq.asq_last_status);
7099                         ret = -ENOENT;
7100                         goto err;
7101                 }
7102                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
7103                 vsi->info.valid_sections = 0;
7104                 vsi->seid = ctxt.seid;
7105                 vsi->id = ctxt.vsi_number;
7106         }
7107
7108         /* If macvlan filters already exist, force them to get loaded */
7109         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
7110                 f->changed = true;
7111                 f_count++;
7112         }
7113         if (f_count) {
7114                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
7115                 pf->flags |= I40E_FLAG_FILTER_SYNC;
7116         }
7117
7118         /* Update VSI BW information */
7119         ret = i40e_vsi_get_bw_info(vsi);
7120         if (ret) {
7121                 dev_info(&pf->pdev->dev,
7122                          "couldn't get vsi bw info, err %d, aq_err %d\n",
7123                          ret, pf->hw.aq.asq_last_status);
7124                 /* VSI is already added so not tearing that up */
7125                 ret = 0;
7126         }
7127
7128 err:
7129         return ret;
7130 }
7131
7132 /**
7133  * i40e_vsi_release - Delete a VSI and free its resources
7134  * @vsi: the VSI being removed
7135  *
7136  * Returns 0 on success or < 0 on error
7137  **/
7138 int i40e_vsi_release(struct i40e_vsi *vsi)
7139 {
7140         struct i40e_mac_filter *f, *ftmp;
7141         struct i40e_veb *veb = NULL;
7142         struct i40e_pf *pf;
7143         u16 uplink_seid;
7144         int i, n;
7145
7146         pf = vsi->back;
7147
7148         /* release of a VEB-owner or last VSI is not allowed */
7149         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
7150                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
7151                          vsi->seid, vsi->uplink_seid);
7152                 return -ENODEV;
7153         }
7154         if (vsi == pf->vsi[pf->lan_vsi] &&
7155             !test_bit(__I40E_DOWN, &pf->state)) {
7156                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
7157                 return -ENODEV;
7158         }
7159
7160         uplink_seid = vsi->uplink_seid;
7161         if (vsi->type != I40E_VSI_SRIOV) {
7162                 if (vsi->netdev_registered) {
7163                         vsi->netdev_registered = false;
7164                         if (vsi->netdev) {
7165                                 /* results in a call to i40e_close() */
7166                                 unregister_netdev(vsi->netdev);
7167                         }
7168                 } else {
7169                         i40e_vsi_close(vsi);
7170                 }
7171                 i40e_vsi_disable_irq(vsi);
7172         }
7173
7174         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list)
7175                 i40e_del_filter(vsi, f->macaddr, f->vlan,
7176                                 f->is_vf, f->is_netdev);
7177         i40e_sync_vsi_filters(vsi);
7178
7179         i40e_vsi_delete(vsi);
7180         i40e_vsi_free_q_vectors(vsi);
7181         if (vsi->netdev) {
7182                 free_netdev(vsi->netdev);
7183                 vsi->netdev = NULL;
7184         }
7185         i40e_vsi_clear_rings(vsi);
7186         i40e_vsi_clear(vsi);
7187
7188         /* If this was the last thing on the VEB, except for the
7189          * controlling VSI, remove the VEB, which puts the controlling
7190          * VSI onto the next level down in the switch.
7191          *
7192          * Well, okay, there's one more exception here: don't remove
7193          * the orphan VEBs yet.  We'll wait for an explicit remove request
7194          * from up the network stack.
7195          */
7196         for (n = 0, i = 0; i < pf->hw.func_caps.num_vsis; i++) {
7197                 if (pf->vsi[i] &&
7198                     pf->vsi[i]->uplink_seid == uplink_seid &&
7199                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
7200                         n++;      /* count the VSIs */
7201                 }
7202         }
7203         for (i = 0; i < I40E_MAX_VEB; i++) {
7204                 if (!pf->veb[i])
7205                         continue;
7206                 if (pf->veb[i]->uplink_seid == uplink_seid)
7207                         n++;     /* count the VEBs */
7208                 if (pf->veb[i]->seid == uplink_seid)
7209                         veb = pf->veb[i];
7210         }
7211         if (n == 0 && veb && veb->uplink_seid != 0)
7212                 i40e_veb_release(veb);
7213
7214         return 0;
7215 }
7216
7217 /**
7218  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
7219  * @vsi: ptr to the VSI
7220  *
7221  * This should only be called after i40e_vsi_mem_alloc() which allocates the
7222  * corresponding SW VSI structure and initializes num_queue_pairs for the
7223  * newly allocated VSI.
7224  *
7225  * Returns 0 on success or negative on failure
7226  **/
7227 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
7228 {
7229         int ret = -ENOENT;
7230         struct i40e_pf *pf = vsi->back;
7231
7232         if (vsi->q_vectors[0]) {
7233                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
7234                          vsi->seid);
7235                 return -EEXIST;
7236         }
7237
7238         if (vsi->base_vector) {
7239                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
7240                          vsi->seid, vsi->base_vector);
7241                 return -EEXIST;
7242         }
7243
7244         ret = i40e_vsi_alloc_q_vectors(vsi);
7245         if (ret) {
7246                 dev_info(&pf->pdev->dev,
7247                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
7248                          vsi->num_q_vectors, vsi->seid, ret);
7249                 vsi->num_q_vectors = 0;
7250                 goto vector_setup_out;
7251         }
7252
7253         if (vsi->num_q_vectors)
7254                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
7255                                                  vsi->num_q_vectors, vsi->idx);
7256         if (vsi->base_vector < 0) {
7257                 dev_info(&pf->pdev->dev,
7258                          "failed to get queue tracking for VSI %d, err=%d\n",
7259                          vsi->seid, vsi->base_vector);
7260                 i40e_vsi_free_q_vectors(vsi);
7261                 ret = -ENOENT;
7262                 goto vector_setup_out;
7263         }
7264
7265 vector_setup_out:
7266         return ret;
7267 }
7268
7269 /**
7270  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
7271  * @vsi: pointer to the vsi.
7272  *
7273  * This re-allocates a vsi's queue resources.
7274  *
7275  * Returns pointer to the successfully allocated and configured VSI sw struct
7276  * on success, otherwise returns NULL on failure.
7277  **/
7278 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
7279 {
7280         struct i40e_pf *pf = vsi->back;
7281         u8 enabled_tc;
7282         int ret;
7283
7284         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
7285         i40e_vsi_clear_rings(vsi);
7286
7287         i40e_vsi_free_arrays(vsi, false);
7288         i40e_set_num_rings_in_vsi(vsi);
7289         ret = i40e_vsi_alloc_arrays(vsi, false);
7290         if (ret)
7291                 goto err_vsi;
7292
7293         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx);
7294         if (ret < 0) {
7295                 dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n",
7296                          vsi->seid, ret);
7297                 goto err_vsi;
7298         }
7299         vsi->base_queue = ret;
7300
7301         /* Update the FW view of the VSI. Force a reset of TC and queue
7302          * layout configurations.
7303          */
7304         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
7305         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
7306         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
7307         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
7308
7309         /* assign it some queues */
7310         ret = i40e_alloc_rings(vsi);
7311         if (ret)
7312                 goto err_rings;
7313
7314         /* map all of the rings to the q_vectors */
7315         i40e_vsi_map_rings_to_vectors(vsi);
7316         return vsi;
7317
7318 err_rings:
7319         i40e_vsi_free_q_vectors(vsi);
7320         if (vsi->netdev_registered) {
7321                 vsi->netdev_registered = false;
7322                 unregister_netdev(vsi->netdev);
7323                 free_netdev(vsi->netdev);
7324                 vsi->netdev = NULL;
7325         }
7326         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
7327 err_vsi:
7328         i40e_vsi_clear(vsi);
7329         return NULL;
7330 }
7331
7332 /**
7333  * i40e_vsi_setup - Set up a VSI by a given type
7334  * @pf: board private structure
7335  * @type: VSI type
7336  * @uplink_seid: the switch element to link to
7337  * @param1: usage depends upon VSI type. For VF types, indicates VF id
7338  *
7339  * This allocates the sw VSI structure and its queue resources, then add a VSI
7340  * to the identified VEB.
7341  *
7342  * Returns pointer to the successfully allocated and configure VSI sw struct on
7343  * success, otherwise returns NULL on failure.
7344  **/
7345 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
7346                                 u16 uplink_seid, u32 param1)
7347 {
7348         struct i40e_vsi *vsi = NULL;
7349         struct i40e_veb *veb = NULL;
7350         int ret, i;
7351         int v_idx;
7352
7353         /* The requested uplink_seid must be either
7354          *     - the PF's port seid
7355          *              no VEB is needed because this is the PF
7356          *              or this is a Flow Director special case VSI
7357          *     - seid of an existing VEB
7358          *     - seid of a VSI that owns an existing VEB
7359          *     - seid of a VSI that doesn't own a VEB
7360          *              a new VEB is created and the VSI becomes the owner
7361          *     - seid of the PF VSI, which is what creates the first VEB
7362          *              this is a special case of the previous
7363          *
7364          * Find which uplink_seid we were given and create a new VEB if needed
7365          */
7366         for (i = 0; i < I40E_MAX_VEB; i++) {
7367                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
7368                         veb = pf->veb[i];
7369                         break;
7370                 }
7371         }
7372
7373         if (!veb && uplink_seid != pf->mac_seid) {
7374
7375                 for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
7376                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
7377                                 vsi = pf->vsi[i];
7378                                 break;
7379                         }
7380                 }
7381                 if (!vsi) {
7382                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
7383                                  uplink_seid);
7384                         return NULL;
7385                 }
7386
7387                 if (vsi->uplink_seid == pf->mac_seid)
7388                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
7389                                              vsi->tc_config.enabled_tc);
7390                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
7391                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
7392                                              vsi->tc_config.enabled_tc);
7393
7394                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
7395                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
7396                                 veb = pf->veb[i];
7397                 }
7398                 if (!veb) {
7399                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
7400                         return NULL;
7401                 }
7402
7403                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
7404                 uplink_seid = veb->seid;
7405         }
7406
7407         /* get vsi sw struct */
7408         v_idx = i40e_vsi_mem_alloc(pf, type);
7409         if (v_idx < 0)
7410                 goto err_alloc;
7411         vsi = pf->vsi[v_idx];
7412         if (!vsi)
7413                 goto err_alloc;
7414         vsi->type = type;
7415         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
7416
7417         if (type == I40E_VSI_MAIN)
7418                 pf->lan_vsi = v_idx;
7419         else if (type == I40E_VSI_SRIOV)
7420                 vsi->vf_id = param1;
7421         /* assign it some queues */
7422         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs,
7423                                 vsi->idx);
7424         if (ret < 0) {
7425                 dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n",
7426                          vsi->seid, ret);
7427                 goto err_vsi;
7428         }
7429         vsi->base_queue = ret;
7430
7431         /* get a VSI from the hardware */
7432         vsi->uplink_seid = uplink_seid;
7433         ret = i40e_add_vsi(vsi);
7434         if (ret)
7435                 goto err_vsi;
7436
7437         switch (vsi->type) {
7438         /* setup the netdev if needed */
7439         case I40E_VSI_MAIN:
7440         case I40E_VSI_VMDQ2:
7441                 ret = i40e_config_netdev(vsi);
7442                 if (ret)
7443                         goto err_netdev;
7444                 ret = register_netdev(vsi->netdev);
7445                 if (ret)
7446                         goto err_netdev;
7447                 vsi->netdev_registered = true;
7448                 netif_carrier_off(vsi->netdev);
7449 #ifdef CONFIG_I40E_DCB
7450                 /* Setup DCB netlink interface */
7451                 i40e_dcbnl_setup(vsi);
7452 #endif /* CONFIG_I40E_DCB */
7453                 /* fall through */
7454
7455         case I40E_VSI_FDIR:
7456                 /* set up vectors and rings if needed */
7457                 ret = i40e_vsi_setup_vectors(vsi);
7458                 if (ret)
7459                         goto err_msix;
7460
7461                 ret = i40e_alloc_rings(vsi);
7462                 if (ret)
7463                         goto err_rings;
7464
7465                 /* map all of the rings to the q_vectors */
7466                 i40e_vsi_map_rings_to_vectors(vsi);
7467
7468                 i40e_vsi_reset_stats(vsi);
7469                 break;
7470
7471         default:
7472                 /* no netdev or rings for the other VSI types */
7473                 break;
7474         }
7475
7476         return vsi;
7477
7478 err_rings:
7479         i40e_vsi_free_q_vectors(vsi);
7480 err_msix:
7481         if (vsi->netdev_registered) {
7482                 vsi->netdev_registered = false;
7483                 unregister_netdev(vsi->netdev);
7484                 free_netdev(vsi->netdev);
7485                 vsi->netdev = NULL;
7486         }
7487 err_netdev:
7488         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
7489 err_vsi:
7490         i40e_vsi_clear(vsi);
7491 err_alloc:
7492         return NULL;
7493 }
7494
7495 /**
7496  * i40e_veb_get_bw_info - Query VEB BW information
7497  * @veb: the veb to query
7498  *
7499  * Query the Tx scheduler BW configuration data for given VEB
7500  **/
7501 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
7502 {
7503         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
7504         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
7505         struct i40e_pf *pf = veb->pf;
7506         struct i40e_hw *hw = &pf->hw;
7507         u32 tc_bw_max;
7508         int ret = 0;
7509         int i;
7510
7511         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
7512                                                   &bw_data, NULL);
7513         if (ret) {
7514                 dev_info(&pf->pdev->dev,
7515                          "query veb bw config failed, aq_err=%d\n",
7516                          hw->aq.asq_last_status);
7517                 goto out;
7518         }
7519
7520         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
7521                                                    &ets_data, NULL);
7522         if (ret) {
7523                 dev_info(&pf->pdev->dev,
7524                          "query veb bw ets config failed, aq_err=%d\n",
7525                          hw->aq.asq_last_status);
7526                 goto out;
7527         }
7528
7529         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
7530         veb->bw_max_quanta = ets_data.tc_bw_max;
7531         veb->is_abs_credits = bw_data.absolute_credits_enable;
7532         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
7533                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
7534         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7535                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
7536                 veb->bw_tc_limit_credits[i] =
7537                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
7538                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
7539         }
7540
7541 out:
7542         return ret;
7543 }
7544
7545 /**
7546  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
7547  * @pf: board private structure
7548  *
7549  * On error: returns error code (negative)
7550  * On success: returns vsi index in PF (positive)
7551  **/
7552 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
7553 {
7554         int ret = -ENOENT;
7555         struct i40e_veb *veb;
7556         int i;
7557
7558         /* Need to protect the allocation of switch elements at the PF level */
7559         mutex_lock(&pf->switch_mutex);
7560
7561         /* VEB list may be fragmented if VEB creation/destruction has
7562          * been happening.  We can afford to do a quick scan to look
7563          * for any free slots in the list.
7564          *
7565          * find next empty veb slot, looping back around if necessary
7566          */
7567         i = 0;
7568         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
7569                 i++;
7570         if (i >= I40E_MAX_VEB) {
7571                 ret = -ENOMEM;
7572                 goto err_alloc_veb;  /* out of VEB slots! */
7573         }
7574
7575         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
7576         if (!veb) {
7577                 ret = -ENOMEM;
7578                 goto err_alloc_veb;
7579         }
7580         veb->pf = pf;
7581         veb->idx = i;
7582         veb->enabled_tc = 1;
7583
7584         pf->veb[i] = veb;
7585         ret = i;
7586 err_alloc_veb:
7587         mutex_unlock(&pf->switch_mutex);
7588         return ret;
7589 }
7590
7591 /**
7592  * i40e_switch_branch_release - Delete a branch of the switch tree
7593  * @branch: where to start deleting
7594  *
7595  * This uses recursion to find the tips of the branch to be
7596  * removed, deleting until we get back to and can delete this VEB.
7597  **/
7598 static void i40e_switch_branch_release(struct i40e_veb *branch)
7599 {
7600         struct i40e_pf *pf = branch->pf;
7601         u16 branch_seid = branch->seid;
7602         u16 veb_idx = branch->idx;
7603         int i;
7604
7605         /* release any VEBs on this VEB - RECURSION */
7606         for (i = 0; i < I40E_MAX_VEB; i++) {
7607                 if (!pf->veb[i])
7608                         continue;
7609                 if (pf->veb[i]->uplink_seid == branch->seid)
7610                         i40e_switch_branch_release(pf->veb[i]);
7611         }
7612
7613         /* Release the VSIs on this VEB, but not the owner VSI.
7614          *
7615          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
7616          *       the VEB itself, so don't use (*branch) after this loop.
7617          */
7618         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
7619                 if (!pf->vsi[i])
7620                         continue;
7621                 if (pf->vsi[i]->uplink_seid == branch_seid &&
7622                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
7623                         i40e_vsi_release(pf->vsi[i]);
7624                 }
7625         }
7626
7627         /* There's one corner case where the VEB might not have been
7628          * removed, so double check it here and remove it if needed.
7629          * This case happens if the veb was created from the debugfs
7630          * commands and no VSIs were added to it.
7631          */
7632         if (pf->veb[veb_idx])
7633                 i40e_veb_release(pf->veb[veb_idx]);
7634 }
7635
7636 /**
7637  * i40e_veb_clear - remove veb struct
7638  * @veb: the veb to remove
7639  **/
7640 static void i40e_veb_clear(struct i40e_veb *veb)
7641 {
7642         if (!veb)
7643                 return;
7644
7645         if (veb->pf) {
7646                 struct i40e_pf *pf = veb->pf;
7647
7648                 mutex_lock(&pf->switch_mutex);
7649                 if (pf->veb[veb->idx] == veb)
7650                         pf->veb[veb->idx] = NULL;
7651                 mutex_unlock(&pf->switch_mutex);
7652         }
7653
7654         kfree(veb);
7655 }
7656
7657 /**
7658  * i40e_veb_release - Delete a VEB and free its resources
7659  * @veb: the VEB being removed
7660  **/
7661 void i40e_veb_release(struct i40e_veb *veb)
7662 {
7663         struct i40e_vsi *vsi = NULL;
7664         struct i40e_pf *pf;
7665         int i, n = 0;
7666
7667         pf = veb->pf;
7668
7669         /* find the remaining VSI and check for extras */
7670         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
7671                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
7672                         n++;
7673                         vsi = pf->vsi[i];
7674                 }
7675         }
7676         if (n != 1) {
7677                 dev_info(&pf->pdev->dev,
7678                          "can't remove VEB %d with %d VSIs left\n",
7679                          veb->seid, n);
7680                 return;
7681         }
7682
7683         /* move the remaining VSI to uplink veb */
7684         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
7685         if (veb->uplink_seid) {
7686                 vsi->uplink_seid = veb->uplink_seid;
7687                 if (veb->uplink_seid == pf->mac_seid)
7688                         vsi->veb_idx = I40E_NO_VEB;
7689                 else
7690                         vsi->veb_idx = veb->veb_idx;
7691         } else {
7692                 /* floating VEB */
7693                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
7694                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
7695         }
7696
7697         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
7698         i40e_veb_clear(veb);
7699 }
7700
7701 /**
7702  * i40e_add_veb - create the VEB in the switch
7703  * @veb: the VEB to be instantiated
7704  * @vsi: the controlling VSI
7705  **/
7706 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
7707 {
7708         bool is_default = false;
7709         bool is_cloud = false;
7710         int ret;
7711
7712         /* get a VEB from the hardware */
7713         ret = i40e_aq_add_veb(&veb->pf->hw, veb->uplink_seid, vsi->seid,
7714                               veb->enabled_tc, is_default,
7715                               is_cloud, &veb->seid, NULL);
7716         if (ret) {
7717                 dev_info(&veb->pf->pdev->dev,
7718                          "couldn't add VEB, err %d, aq_err %d\n",
7719                          ret, veb->pf->hw.aq.asq_last_status);
7720                 return -EPERM;
7721         }
7722
7723         /* get statistics counter */
7724         ret = i40e_aq_get_veb_parameters(&veb->pf->hw, veb->seid, NULL, NULL,
7725                                          &veb->stats_idx, NULL, NULL, NULL);
7726         if (ret) {
7727                 dev_info(&veb->pf->pdev->dev,
7728                          "couldn't get VEB statistics idx, err %d, aq_err %d\n",
7729                          ret, veb->pf->hw.aq.asq_last_status);
7730                 return -EPERM;
7731         }
7732         ret = i40e_veb_get_bw_info(veb);
7733         if (ret) {
7734                 dev_info(&veb->pf->pdev->dev,
7735                          "couldn't get VEB bw info, err %d, aq_err %d\n",
7736                          ret, veb->pf->hw.aq.asq_last_status);
7737                 i40e_aq_delete_element(&veb->pf->hw, veb->seid, NULL);
7738                 return -ENOENT;
7739         }
7740
7741         vsi->uplink_seid = veb->seid;
7742         vsi->veb_idx = veb->idx;
7743         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
7744
7745         return 0;
7746 }
7747
7748 /**
7749  * i40e_veb_setup - Set up a VEB
7750  * @pf: board private structure
7751  * @flags: VEB setup flags
7752  * @uplink_seid: the switch element to link to
7753  * @vsi_seid: the initial VSI seid
7754  * @enabled_tc: Enabled TC bit-map
7755  *
7756  * This allocates the sw VEB structure and links it into the switch
7757  * It is possible and legal for this to be a duplicate of an already
7758  * existing VEB.  It is also possible for both uplink and vsi seids
7759  * to be zero, in order to create a floating VEB.
7760  *
7761  * Returns pointer to the successfully allocated VEB sw struct on
7762  * success, otherwise returns NULL on failure.
7763  **/
7764 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
7765                                 u16 uplink_seid, u16 vsi_seid,
7766                                 u8 enabled_tc)
7767 {
7768         struct i40e_veb *veb, *uplink_veb = NULL;
7769         int vsi_idx, veb_idx;
7770         int ret;
7771
7772         /* if one seid is 0, the other must be 0 to create a floating relay */
7773         if ((uplink_seid == 0 || vsi_seid == 0) &&
7774             (uplink_seid + vsi_seid != 0)) {
7775                 dev_info(&pf->pdev->dev,
7776                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
7777                          uplink_seid, vsi_seid);
7778                 return NULL;
7779         }
7780
7781         /* make sure there is such a vsi and uplink */
7782         for (vsi_idx = 0; vsi_idx < pf->hw.func_caps.num_vsis; vsi_idx++)
7783                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
7784                         break;
7785         if (vsi_idx >= pf->hw.func_caps.num_vsis && vsi_seid != 0) {
7786                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
7787                          vsi_seid);
7788                 return NULL;
7789         }
7790
7791         if (uplink_seid && uplink_seid != pf->mac_seid) {
7792                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
7793                         if (pf->veb[veb_idx] &&
7794                             pf->veb[veb_idx]->seid == uplink_seid) {
7795                                 uplink_veb = pf->veb[veb_idx];
7796                                 break;
7797                         }
7798                 }
7799                 if (!uplink_veb) {
7800                         dev_info(&pf->pdev->dev,
7801                                  "uplink seid %d not found\n", uplink_seid);
7802                         return NULL;
7803                 }
7804         }
7805
7806         /* get veb sw struct */
7807         veb_idx = i40e_veb_mem_alloc(pf);
7808         if (veb_idx < 0)
7809                 goto err_alloc;
7810         veb = pf->veb[veb_idx];
7811         veb->flags = flags;
7812         veb->uplink_seid = uplink_seid;
7813         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
7814         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
7815
7816         /* create the VEB in the switch */
7817         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
7818         if (ret)
7819                 goto err_veb;
7820         if (vsi_idx == pf->lan_vsi)
7821                 pf->lan_veb = veb->idx;
7822
7823         return veb;
7824
7825 err_veb:
7826         i40e_veb_clear(veb);
7827 err_alloc:
7828         return NULL;
7829 }
7830
7831 /**
7832  * i40e_setup_pf_switch_element - set pf vars based on switch type
7833  * @pf: board private structure
7834  * @ele: element we are building info from
7835  * @num_reported: total number of elements
7836  * @printconfig: should we print the contents
7837  *
7838  * helper function to assist in extracting a few useful SEID values.
7839  **/
7840 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
7841                                 struct i40e_aqc_switch_config_element_resp *ele,
7842                                 u16 num_reported, bool printconfig)
7843 {
7844         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
7845         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
7846         u8 element_type = ele->element_type;
7847         u16 seid = le16_to_cpu(ele->seid);
7848
7849         if (printconfig)
7850                 dev_info(&pf->pdev->dev,
7851                          "type=%d seid=%d uplink=%d downlink=%d\n",
7852                          element_type, seid, uplink_seid, downlink_seid);
7853
7854         switch (element_type) {
7855         case I40E_SWITCH_ELEMENT_TYPE_MAC:
7856                 pf->mac_seid = seid;
7857                 break;
7858         case I40E_SWITCH_ELEMENT_TYPE_VEB:
7859                 /* Main VEB? */
7860                 if (uplink_seid != pf->mac_seid)
7861                         break;
7862                 if (pf->lan_veb == I40E_NO_VEB) {
7863                         int v;
7864
7865                         /* find existing or else empty VEB */
7866                         for (v = 0; v < I40E_MAX_VEB; v++) {
7867                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
7868                                         pf->lan_veb = v;
7869                                         break;
7870                                 }
7871                         }
7872                         if (pf->lan_veb == I40E_NO_VEB) {
7873                                 v = i40e_veb_mem_alloc(pf);
7874                                 if (v < 0)
7875                                         break;
7876                                 pf->lan_veb = v;
7877                         }
7878                 }
7879
7880                 pf->veb[pf->lan_veb]->seid = seid;
7881                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
7882                 pf->veb[pf->lan_veb]->pf = pf;
7883                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
7884                 break;
7885         case I40E_SWITCH_ELEMENT_TYPE_VSI:
7886                 if (num_reported != 1)
7887                         break;
7888                 /* This is immediately after a reset so we can assume this is
7889                  * the PF's VSI
7890                  */
7891                 pf->mac_seid = uplink_seid;
7892                 pf->pf_seid = downlink_seid;
7893                 pf->main_vsi_seid = seid;
7894                 if (printconfig)
7895                         dev_info(&pf->pdev->dev,
7896                                  "pf_seid=%d main_vsi_seid=%d\n",
7897                                  pf->pf_seid, pf->main_vsi_seid);
7898                 break;
7899         case I40E_SWITCH_ELEMENT_TYPE_PF:
7900         case I40E_SWITCH_ELEMENT_TYPE_VF:
7901         case I40E_SWITCH_ELEMENT_TYPE_EMP:
7902         case I40E_SWITCH_ELEMENT_TYPE_BMC:
7903         case I40E_SWITCH_ELEMENT_TYPE_PE:
7904         case I40E_SWITCH_ELEMENT_TYPE_PA:
7905                 /* ignore these for now */
7906                 break;
7907         default:
7908                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
7909                          element_type, seid);
7910                 break;
7911         }
7912 }
7913
7914 /**
7915  * i40e_fetch_switch_configuration - Get switch config from firmware
7916  * @pf: board private structure
7917  * @printconfig: should we print the contents
7918  *
7919  * Get the current switch configuration from the device and
7920  * extract a few useful SEID values.
7921  **/
7922 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
7923 {
7924         struct i40e_aqc_get_switch_config_resp *sw_config;
7925         u16 next_seid = 0;
7926         int ret = 0;
7927         u8 *aq_buf;
7928         int i;
7929
7930         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
7931         if (!aq_buf)
7932                 return -ENOMEM;
7933
7934         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
7935         do {
7936                 u16 num_reported, num_total;
7937
7938                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
7939                                                 I40E_AQ_LARGE_BUF,
7940                                                 &next_seid, NULL);
7941                 if (ret) {
7942                         dev_info(&pf->pdev->dev,
7943                                  "get switch config failed %d aq_err=%x\n",
7944                                  ret, pf->hw.aq.asq_last_status);
7945                         kfree(aq_buf);
7946                         return -ENOENT;
7947                 }
7948
7949                 num_reported = le16_to_cpu(sw_config->header.num_reported);
7950                 num_total = le16_to_cpu(sw_config->header.num_total);
7951
7952                 if (printconfig)
7953                         dev_info(&pf->pdev->dev,
7954                                  "header: %d reported %d total\n",
7955                                  num_reported, num_total);
7956
7957                 if (num_reported) {
7958                         int sz = sizeof(*sw_config) * num_reported;
7959
7960                         kfree(pf->sw_config);
7961                         pf->sw_config = kzalloc(sz, GFP_KERNEL);
7962                         if (pf->sw_config)
7963                                 memcpy(pf->sw_config, sw_config, sz);
7964                 }
7965
7966                 for (i = 0; i < num_reported; i++) {
7967                         struct i40e_aqc_switch_config_element_resp *ele =
7968                                 &sw_config->element[i];
7969
7970                         i40e_setup_pf_switch_element(pf, ele, num_reported,
7971                                                      printconfig);
7972                 }
7973         } while (next_seid != 0);
7974
7975         kfree(aq_buf);
7976         return ret;
7977 }
7978
7979 /**
7980  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
7981  * @pf: board private structure
7982  * @reinit: if the Main VSI needs to re-initialized.
7983  *
7984  * Returns 0 on success, negative value on failure
7985  **/
7986 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
7987 {
7988         u32 rxfc = 0, txfc = 0, rxfc_reg;
7989         int ret;
7990
7991         /* find out what's out there already */
7992         ret = i40e_fetch_switch_configuration(pf, false);
7993         if (ret) {
7994                 dev_info(&pf->pdev->dev,
7995                          "couldn't fetch switch config, err %d, aq_err %d\n",
7996                          ret, pf->hw.aq.asq_last_status);
7997                 return ret;
7998         }
7999         i40e_pf_reset_stats(pf);
8000
8001         /* first time setup */
8002         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
8003                 struct i40e_vsi *vsi = NULL;
8004                 u16 uplink_seid;
8005
8006                 /* Set up the PF VSI associated with the PF's main VSI
8007                  * that is already in the HW switch
8008                  */
8009                 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
8010                         uplink_seid = pf->veb[pf->lan_veb]->seid;
8011                 else
8012                         uplink_seid = pf->mac_seid;
8013                 if (pf->lan_vsi == I40E_NO_VSI)
8014                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
8015                 else if (reinit)
8016                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
8017                 if (!vsi) {
8018                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
8019                         i40e_fdir_teardown(pf);
8020                         return -EAGAIN;
8021                 }
8022         } else {
8023                 /* force a reset of TC and queue layout configurations */
8024                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
8025                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
8026                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
8027                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
8028         }
8029         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
8030
8031         i40e_fdir_sb_setup(pf);
8032
8033         /* Setup static PF queue filter control settings */
8034         ret = i40e_setup_pf_filter_control(pf);
8035         if (ret) {
8036                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
8037                          ret);
8038                 /* Failure here should not stop continuing other steps */
8039         }
8040
8041         /* enable RSS in the HW, even for only one queue, as the stack can use
8042          * the hash
8043          */
8044         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
8045                 i40e_config_rss(pf);
8046
8047         /* fill in link information and enable LSE reporting */
8048         i40e_aq_get_link_info(&pf->hw, true, NULL, NULL);
8049         i40e_link_event(pf);
8050
8051         /* Initialize user-specific link properties */
8052         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
8053                                   I40E_AQ_AN_COMPLETED) ? true : false);
8054         /* requested_mode is set in probe or by ethtool */
8055         if (!pf->fc_autoneg_status)
8056                 goto no_autoneg;
8057
8058         if ((pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) &&
8059             (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX))
8060                 pf->hw.fc.current_mode = I40E_FC_FULL;
8061         else if (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
8062                 pf->hw.fc.current_mode = I40E_FC_TX_PAUSE;
8063         else if (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
8064                 pf->hw.fc.current_mode = I40E_FC_RX_PAUSE;
8065         else
8066                 pf->hw.fc.current_mode = I40E_FC_NONE;
8067
8068         /* sync the flow control settings with the auto-neg values */
8069         switch (pf->hw.fc.current_mode) {
8070         case I40E_FC_FULL:
8071                 txfc = 1;
8072                 rxfc = 1;
8073                 break;
8074         case I40E_FC_TX_PAUSE:
8075                 txfc = 1;
8076                 rxfc = 0;
8077                 break;
8078         case I40E_FC_RX_PAUSE:
8079                 txfc = 0;
8080                 rxfc = 1;
8081                 break;
8082         case I40E_FC_NONE:
8083         case I40E_FC_DEFAULT:
8084                 txfc = 0;
8085                 rxfc = 0;
8086                 break;
8087         case I40E_FC_PFC:
8088                 /* TBD */
8089                 break;
8090         /* no default case, we have to handle all possibilities here */
8091         }
8092
8093         wr32(&pf->hw, I40E_PRTDCB_FCCFG, txfc << I40E_PRTDCB_FCCFG_TFCE_SHIFT);
8094
8095         rxfc_reg = rd32(&pf->hw, I40E_PRTDCB_MFLCN) &
8096                    ~I40E_PRTDCB_MFLCN_RFCE_MASK;
8097         rxfc_reg |= (rxfc << I40E_PRTDCB_MFLCN_RFCE_SHIFT);
8098
8099         wr32(&pf->hw, I40E_PRTDCB_MFLCN, rxfc_reg);
8100
8101         goto fc_complete;
8102
8103 no_autoneg:
8104         /* disable L2 flow control, user can turn it on if they wish */
8105         wr32(&pf->hw, I40E_PRTDCB_FCCFG, 0);
8106         wr32(&pf->hw, I40E_PRTDCB_MFLCN, rd32(&pf->hw, I40E_PRTDCB_MFLCN) &
8107                                          ~I40E_PRTDCB_MFLCN_RFCE_MASK);
8108
8109 fc_complete:
8110         i40e_ptp_init(pf);
8111
8112         return ret;
8113 }
8114
8115 /**
8116  * i40e_determine_queue_usage - Work out queue distribution
8117  * @pf: board private structure
8118  **/
8119 static void i40e_determine_queue_usage(struct i40e_pf *pf)
8120 {
8121         int queues_left;
8122
8123         pf->num_lan_qps = 0;
8124
8125         /* Find the max queues to be put into basic use.  We'll always be
8126          * using TC0, whether or not DCB is running, and TC0 will get the
8127          * big RSS set.
8128          */
8129         queues_left = pf->hw.func_caps.num_tx_qp;
8130
8131         if ((queues_left == 1) ||
8132             !(pf->flags & I40E_FLAG_MSIX_ENABLED) ||
8133             !(pf->flags & (I40E_FLAG_RSS_ENABLED | I40E_FLAG_FD_SB_ENABLED |
8134                            I40E_FLAG_DCB_ENABLED))) {
8135                 /* one qp for PF, no queues for anything else */
8136                 queues_left = 0;
8137                 pf->rss_size = pf->num_lan_qps = 1;
8138
8139                 /* make sure all the fancies are disabled */
8140                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
8141                                I40E_FLAG_FD_SB_ENABLED  |
8142                                I40E_FLAG_FD_ATR_ENABLED |
8143                                I40E_FLAG_DCB_ENABLED    |
8144                                I40E_FLAG_SRIOV_ENABLED  |
8145                                I40E_FLAG_VMDQ_ENABLED);
8146         } else {
8147                 /* Not enough queues for all TCs */
8148                 if ((pf->flags & I40E_FLAG_DCB_ENABLED) &&
8149                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
8150                         pf->flags &= ~I40E_FLAG_DCB_ENABLED;
8151                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
8152                 }
8153                 pf->num_lan_qps = pf->rss_size_max;
8154                 queues_left -= pf->num_lan_qps;
8155         }
8156
8157         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8158                 if (queues_left > 1) {
8159                         queues_left -= 1; /* save 1 queue for FD */
8160                 } else {
8161                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8162                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
8163                 }
8164         }
8165
8166         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
8167             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
8168                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
8169                                         (queues_left / pf->num_vf_qps));
8170                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
8171         }
8172
8173         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
8174             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
8175                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
8176                                           (queues_left / pf->num_vmdq_qps));
8177                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
8178         }
8179
8180         pf->queues_left = queues_left;
8181 }
8182
8183 /**
8184  * i40e_setup_pf_filter_control - Setup PF static filter control
8185  * @pf: PF to be setup
8186  *
8187  * i40e_setup_pf_filter_control sets up a pf's initial filter control
8188  * settings. If PE/FCoE are enabled then it will also set the per PF
8189  * based filter sizes required for them. It also enables Flow director,
8190  * ethertype and macvlan type filter settings for the pf.
8191  *
8192  * Returns 0 on success, negative on failure
8193  **/
8194 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
8195 {
8196         struct i40e_filter_control_settings *settings = &pf->filter_settings;
8197
8198         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
8199
8200         /* Flow Director is enabled */
8201         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
8202                 settings->enable_fdir = true;
8203
8204         /* Ethtype and MACVLAN filters enabled for PF */
8205         settings->enable_ethtype = true;
8206         settings->enable_macvlan = true;
8207
8208         if (i40e_set_filter_control(&pf->hw, settings))
8209                 return -ENOENT;
8210
8211         return 0;
8212 }
8213
8214 #define INFO_STRING_LEN 255
8215 static void i40e_print_features(struct i40e_pf *pf)
8216 {
8217         struct i40e_hw *hw = &pf->hw;
8218         char *buf, *string;
8219
8220         string = kzalloc(INFO_STRING_LEN, GFP_KERNEL);
8221         if (!string) {
8222                 dev_err(&pf->pdev->dev, "Features string allocation failed\n");
8223                 return;
8224         }
8225
8226         buf = string;
8227
8228         buf += sprintf(string, "Features: PF-id[%d] ", hw->pf_id);
8229 #ifdef CONFIG_PCI_IOV
8230         buf += sprintf(buf, "VFs: %d ", pf->num_req_vfs);
8231 #endif
8232         buf += sprintf(buf, "VSIs: %d QP: %d ", pf->hw.func_caps.num_vsis,
8233                        pf->vsi[pf->lan_vsi]->num_queue_pairs);
8234
8235         if (pf->flags & I40E_FLAG_RSS_ENABLED)
8236                 buf += sprintf(buf, "RSS ");
8237         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
8238                 buf += sprintf(buf, "FD_ATR ");
8239         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8240                 buf += sprintf(buf, "FD_SB ");
8241                 buf += sprintf(buf, "NTUPLE ");
8242         }
8243         if (pf->flags & I40E_FLAG_DCB_ENABLED)
8244                 buf += sprintf(buf, "DCB ");
8245         if (pf->flags & I40E_FLAG_PTP)
8246                 buf += sprintf(buf, "PTP ");
8247
8248         BUG_ON(buf > (string + INFO_STRING_LEN));
8249         dev_info(&pf->pdev->dev, "%s\n", string);
8250         kfree(string);
8251 }
8252
8253 /**
8254  * i40e_probe - Device initialization routine
8255  * @pdev: PCI device information struct
8256  * @ent: entry in i40e_pci_tbl
8257  *
8258  * i40e_probe initializes a pf identified by a pci_dev structure.
8259  * The OS initialization, configuring of the pf private structure,
8260  * and a hardware reset occur.
8261  *
8262  * Returns 0 on success, negative on failure
8263  **/
8264 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
8265 {
8266         struct i40e_pf *pf;
8267         struct i40e_hw *hw;
8268         static u16 pfs_found;
8269         u16 link_status;
8270         int err = 0;
8271         u32 len;
8272         u32 i;
8273
8274         err = pci_enable_device_mem(pdev);
8275         if (err)
8276                 return err;
8277
8278         /* set up for high or low dma */
8279         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
8280         if (err) {
8281                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
8282                 if (err) {
8283                         dev_err(&pdev->dev,
8284                                 "DMA configuration failed: 0x%x\n", err);
8285                         goto err_dma;
8286                 }
8287         }
8288
8289         /* set up pci connections */
8290         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
8291                                            IORESOURCE_MEM), i40e_driver_name);
8292         if (err) {
8293                 dev_info(&pdev->dev,
8294                          "pci_request_selected_regions failed %d\n", err);
8295                 goto err_pci_reg;
8296         }
8297
8298         pci_enable_pcie_error_reporting(pdev);
8299         pci_set_master(pdev);
8300
8301         /* Now that we have a PCI connection, we need to do the
8302          * low level device setup.  This is primarily setting up
8303          * the Admin Queue structures and then querying for the
8304          * device's current profile information.
8305          */
8306         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
8307         if (!pf) {
8308                 err = -ENOMEM;
8309                 goto err_pf_alloc;
8310         }
8311         pf->next_vsi = 0;
8312         pf->pdev = pdev;
8313         set_bit(__I40E_DOWN, &pf->state);
8314
8315         hw = &pf->hw;
8316         hw->back = pf;
8317         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
8318                               pci_resource_len(pdev, 0));
8319         if (!hw->hw_addr) {
8320                 err = -EIO;
8321                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
8322                          (unsigned int)pci_resource_start(pdev, 0),
8323                          (unsigned int)pci_resource_len(pdev, 0), err);
8324                 goto err_ioremap;
8325         }
8326         hw->vendor_id = pdev->vendor;
8327         hw->device_id = pdev->device;
8328         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
8329         hw->subsystem_vendor_id = pdev->subsystem_vendor;
8330         hw->subsystem_device_id = pdev->subsystem_device;
8331         hw->bus.device = PCI_SLOT(pdev->devfn);
8332         hw->bus.func = PCI_FUNC(pdev->devfn);
8333         pf->instance = pfs_found;
8334
8335         /* do a special CORER for clearing PXE mode once at init */
8336         if (hw->revision_id == 0 &&
8337             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
8338                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
8339                 i40e_flush(hw);
8340                 msleep(200);
8341                 pf->corer_count++;
8342
8343                 i40e_clear_pxe_mode(hw);
8344         }
8345
8346         /* Reset here to make sure all is clean and to define PF 'n' */
8347         err = i40e_pf_reset(hw);
8348         if (err) {
8349                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
8350                 goto err_pf_reset;
8351         }
8352         pf->pfr_count++;
8353
8354         hw->aq.num_arq_entries = I40E_AQ_LEN;
8355         hw->aq.num_asq_entries = I40E_AQ_LEN;
8356         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
8357         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
8358         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
8359         snprintf(pf->misc_int_name, sizeof(pf->misc_int_name) - 1,
8360                  "%s-pf%d:misc",
8361                  dev_driver_string(&pf->pdev->dev), pf->hw.pf_id);
8362
8363         err = i40e_init_shared_code(hw);
8364         if (err) {
8365                 dev_info(&pdev->dev, "init_shared_code failed: %d\n", err);
8366                 goto err_pf_reset;
8367         }
8368
8369         /* set up a default setting for link flow control */
8370         pf->hw.fc.requested_mode = I40E_FC_NONE;
8371
8372         err = i40e_init_adminq(hw);
8373         dev_info(&pdev->dev, "%s\n", i40e_fw_version_str(hw));
8374         if (err) {
8375                 dev_info(&pdev->dev,
8376                          "init_adminq failed: %d expecting API %02x.%02x\n",
8377                          err,
8378                          I40E_FW_API_VERSION_MAJOR, I40E_FW_API_VERSION_MINOR);
8379                 goto err_pf_reset;
8380         }
8381
8382         i40e_verify_eeprom(pf);
8383
8384         /* Rev 0 hardware was never productized */
8385         if (hw->revision_id < 1)
8386                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
8387
8388         i40e_clear_pxe_mode(hw);
8389         err = i40e_get_capabilities(pf);
8390         if (err)
8391                 goto err_adminq_setup;
8392
8393         err = i40e_sw_init(pf);
8394         if (err) {
8395                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
8396                 goto err_sw_init;
8397         }
8398
8399         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
8400                                 hw->func_caps.num_rx_qp,
8401                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
8402         if (err) {
8403                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
8404                 goto err_init_lan_hmc;
8405         }
8406
8407         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
8408         if (err) {
8409                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
8410                 err = -ENOENT;
8411                 goto err_configure_lan_hmc;
8412         }
8413
8414         i40e_get_mac_addr(hw, hw->mac.addr);
8415         if (!is_valid_ether_addr(hw->mac.addr)) {
8416                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
8417                 err = -EIO;
8418                 goto err_mac_addr;
8419         }
8420         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
8421         memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
8422
8423         pci_set_drvdata(pdev, pf);
8424         pci_save_state(pdev);
8425 #ifdef CONFIG_I40E_DCB
8426         err = i40e_init_pf_dcb(pf);
8427         if (err) {
8428                 dev_info(&pdev->dev, "init_pf_dcb failed: %d\n", err);
8429                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
8430                 /* Continue without DCB enabled */
8431         }
8432 #endif /* CONFIG_I40E_DCB */
8433
8434         /* set up periodic task facility */
8435         setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf);
8436         pf->service_timer_period = HZ;
8437
8438         INIT_WORK(&pf->service_task, i40e_service_task);
8439         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
8440         pf->flags |= I40E_FLAG_NEED_LINK_UPDATE;
8441         pf->link_check_timeout = jiffies;
8442
8443         /* WoL defaults to disabled */
8444         pf->wol_en = false;
8445         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
8446
8447         /* set up the main switch operations */
8448         i40e_determine_queue_usage(pf);
8449         i40e_init_interrupt_scheme(pf);
8450
8451         /* Set up the *vsi struct based on the number of VSIs in the HW,
8452          * and set up our local tracking of the MAIN PF vsi.
8453          */
8454         len = sizeof(struct i40e_vsi *) * pf->hw.func_caps.num_vsis;
8455         pf->vsi = kzalloc(len, GFP_KERNEL);
8456         if (!pf->vsi) {
8457                 err = -ENOMEM;
8458                 goto err_switch_setup;
8459         }
8460
8461         err = i40e_setup_pf_switch(pf, false);
8462         if (err) {
8463                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
8464                 goto err_vsis;
8465         }
8466         /* if FDIR VSI was set up, start it now */
8467         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
8468                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
8469                         i40e_vsi_open(pf->vsi[i]);
8470                         break;
8471                 }
8472         }
8473
8474         /* The main driver is (mostly) up and happy. We need to set this state
8475          * before setting up the misc vector or we get a race and the vector
8476          * ends up disabled forever.
8477          */
8478         clear_bit(__I40E_DOWN, &pf->state);
8479
8480         /* In case of MSIX we are going to setup the misc vector right here
8481          * to handle admin queue events etc. In case of legacy and MSI
8482          * the misc functionality and queue processing is combined in
8483          * the same vector and that gets setup at open.
8484          */
8485         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
8486                 err = i40e_setup_misc_vector(pf);
8487                 if (err) {
8488                         dev_info(&pdev->dev,
8489                                  "setup of misc vector failed: %d\n", err);
8490                         goto err_vsis;
8491                 }
8492         }
8493
8494 #ifdef CONFIG_PCI_IOV
8495         /* prep for VF support */
8496         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
8497             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
8498             !test_bit(__I40E_BAD_EEPROM, &pf->state)) {
8499                 u32 val;
8500
8501                 /* disable link interrupts for VFs */
8502                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
8503                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
8504                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
8505                 i40e_flush(hw);
8506
8507                 if (pci_num_vf(pdev)) {
8508                         dev_info(&pdev->dev,
8509                                  "Active VFs found, allocating resources.\n");
8510                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
8511                         if (err)
8512                                 dev_info(&pdev->dev,
8513                                          "Error %d allocating resources for existing VFs\n",
8514                                          err);
8515                 }
8516         }
8517 #endif /* CONFIG_PCI_IOV */
8518
8519         pfs_found++;
8520
8521         i40e_dbg_pf_init(pf);
8522
8523         /* tell the firmware that we're starting */
8524         i40e_send_version(pf);
8525
8526         /* since everything's happy, start the service_task timer */
8527         mod_timer(&pf->service_timer,
8528                   round_jiffies(jiffies + pf->service_timer_period));
8529
8530         /* Get the negotiated link width and speed from PCI config space */
8531         pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, &link_status);
8532
8533         i40e_set_pci_config_data(hw, link_status);
8534
8535         dev_info(&pdev->dev, "PCI-Express: %s %s\n",
8536                 (hw->bus.speed == i40e_bus_speed_8000 ? "Speed 8.0GT/s" :
8537                  hw->bus.speed == i40e_bus_speed_5000 ? "Speed 5.0GT/s" :
8538                  hw->bus.speed == i40e_bus_speed_2500 ? "Speed 2.5GT/s" :
8539                  "Unknown"),
8540                 (hw->bus.width == i40e_bus_width_pcie_x8 ? "Width x8" :
8541                  hw->bus.width == i40e_bus_width_pcie_x4 ? "Width x4" :
8542                  hw->bus.width == i40e_bus_width_pcie_x2 ? "Width x2" :
8543                  hw->bus.width == i40e_bus_width_pcie_x1 ? "Width x1" :
8544                  "Unknown"));
8545
8546         if (hw->bus.width < i40e_bus_width_pcie_x8 ||
8547             hw->bus.speed < i40e_bus_speed_8000) {
8548                 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
8549                 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
8550         }
8551
8552         /* print a string summarizing features */
8553         i40e_print_features(pf);
8554
8555         return 0;
8556
8557         /* Unwind what we've done if something failed in the setup */
8558 err_vsis:
8559         set_bit(__I40E_DOWN, &pf->state);
8560         i40e_clear_interrupt_scheme(pf);
8561         kfree(pf->vsi);
8562 err_switch_setup:
8563         i40e_reset_interrupt_capability(pf);
8564         del_timer_sync(&pf->service_timer);
8565 err_mac_addr:
8566 err_configure_lan_hmc:
8567         (void)i40e_shutdown_lan_hmc(hw);
8568 err_init_lan_hmc:
8569         kfree(pf->qp_pile);
8570         kfree(pf->irq_pile);
8571 err_sw_init:
8572 err_adminq_setup:
8573         (void)i40e_shutdown_adminq(hw);
8574 err_pf_reset:
8575         iounmap(hw->hw_addr);
8576 err_ioremap:
8577         kfree(pf);
8578 err_pf_alloc:
8579         pci_disable_pcie_error_reporting(pdev);
8580         pci_release_selected_regions(pdev,
8581                                      pci_select_bars(pdev, IORESOURCE_MEM));
8582 err_pci_reg:
8583 err_dma:
8584         pci_disable_device(pdev);
8585         return err;
8586 }
8587
8588 /**
8589  * i40e_remove - Device removal routine
8590  * @pdev: PCI device information struct
8591  *
8592  * i40e_remove is called by the PCI subsystem to alert the driver
8593  * that is should release a PCI device.  This could be caused by a
8594  * Hot-Plug event, or because the driver is going to be removed from
8595  * memory.
8596  **/
8597 static void i40e_remove(struct pci_dev *pdev)
8598 {
8599         struct i40e_pf *pf = pci_get_drvdata(pdev);
8600         i40e_status ret_code;
8601         u32 reg;
8602         int i;
8603
8604         i40e_dbg_pf_exit(pf);
8605
8606         i40e_ptp_stop(pf);
8607
8608         /* no more scheduling of any task */
8609         set_bit(__I40E_DOWN, &pf->state);
8610         del_timer_sync(&pf->service_timer);
8611         cancel_work_sync(&pf->service_task);
8612
8613         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
8614                 i40e_free_vfs(pf);
8615                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
8616         }
8617
8618         i40e_fdir_teardown(pf);
8619
8620         /* If there is a switch structure or any orphans, remove them.
8621          * This will leave only the PF's VSI remaining.
8622          */
8623         for (i = 0; i < I40E_MAX_VEB; i++) {
8624                 if (!pf->veb[i])
8625                         continue;
8626
8627                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
8628                     pf->veb[i]->uplink_seid == 0)
8629                         i40e_switch_branch_release(pf->veb[i]);
8630         }
8631
8632         /* Now we can shutdown the PF's VSI, just before we kill
8633          * adminq and hmc.
8634          */
8635         if (pf->vsi[pf->lan_vsi])
8636                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
8637
8638         i40e_stop_misc_vector(pf);
8639         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
8640                 synchronize_irq(pf->msix_entries[0].vector);
8641                 free_irq(pf->msix_entries[0].vector, pf);
8642         }
8643
8644         /* shutdown and destroy the HMC */
8645         if (pf->hw.hmc.hmc_obj) {
8646                 ret_code = i40e_shutdown_lan_hmc(&pf->hw);
8647                 if (ret_code)
8648                         dev_warn(&pdev->dev,
8649                                  "Failed to destroy the HMC resources: %d\n",
8650                                  ret_code);
8651         }
8652
8653         /* shutdown the adminq */
8654         ret_code = i40e_shutdown_adminq(&pf->hw);
8655         if (ret_code)
8656                 dev_warn(&pdev->dev,
8657                          "Failed to destroy the Admin Queue resources: %d\n",
8658                          ret_code);
8659
8660         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
8661         i40e_clear_interrupt_scheme(pf);
8662         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
8663                 if (pf->vsi[i]) {
8664                         i40e_vsi_clear_rings(pf->vsi[i]);
8665                         i40e_vsi_clear(pf->vsi[i]);
8666                         pf->vsi[i] = NULL;
8667                 }
8668         }
8669
8670         for (i = 0; i < I40E_MAX_VEB; i++) {
8671                 kfree(pf->veb[i]);
8672                 pf->veb[i] = NULL;
8673         }
8674
8675         kfree(pf->qp_pile);
8676         kfree(pf->irq_pile);
8677         kfree(pf->sw_config);
8678         kfree(pf->vsi);
8679
8680         /* force a PF reset to clean anything leftover */
8681         reg = rd32(&pf->hw, I40E_PFGEN_CTRL);
8682         wr32(&pf->hw, I40E_PFGEN_CTRL, (reg | I40E_PFGEN_CTRL_PFSWR_MASK));
8683         i40e_flush(&pf->hw);
8684
8685         iounmap(pf->hw.hw_addr);
8686         kfree(pf);
8687         pci_release_selected_regions(pdev,
8688                                      pci_select_bars(pdev, IORESOURCE_MEM));
8689
8690         pci_disable_pcie_error_reporting(pdev);
8691         pci_disable_device(pdev);
8692 }
8693
8694 /**
8695  * i40e_pci_error_detected - warning that something funky happened in PCI land
8696  * @pdev: PCI device information struct
8697  *
8698  * Called to warn that something happened and the error handling steps
8699  * are in progress.  Allows the driver to quiesce things, be ready for
8700  * remediation.
8701  **/
8702 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
8703                                                 enum pci_channel_state error)
8704 {
8705         struct i40e_pf *pf = pci_get_drvdata(pdev);
8706
8707         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
8708
8709         /* shutdown all operations */
8710         if (!test_bit(__I40E_SUSPENDED, &pf->state)) {
8711                 rtnl_lock();
8712                 i40e_prep_for_reset(pf);
8713                 rtnl_unlock();
8714         }
8715
8716         /* Request a slot reset */
8717         return PCI_ERS_RESULT_NEED_RESET;
8718 }
8719
8720 /**
8721  * i40e_pci_error_slot_reset - a PCI slot reset just happened
8722  * @pdev: PCI device information struct
8723  *
8724  * Called to find if the driver can work with the device now that
8725  * the pci slot has been reset.  If a basic connection seems good
8726  * (registers are readable and have sane content) then return a
8727  * happy little PCI_ERS_RESULT_xxx.
8728  **/
8729 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
8730 {
8731         struct i40e_pf *pf = pci_get_drvdata(pdev);
8732         pci_ers_result_t result;
8733         int err;
8734         u32 reg;
8735
8736         dev_info(&pdev->dev, "%s\n", __func__);
8737         if (pci_enable_device_mem(pdev)) {
8738                 dev_info(&pdev->dev,
8739                          "Cannot re-enable PCI device after reset.\n");
8740                 result = PCI_ERS_RESULT_DISCONNECT;
8741         } else {
8742                 pci_set_master(pdev);
8743                 pci_restore_state(pdev);
8744                 pci_save_state(pdev);
8745                 pci_wake_from_d3(pdev, false);
8746
8747                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8748                 if (reg == 0)
8749                         result = PCI_ERS_RESULT_RECOVERED;
8750                 else
8751                         result = PCI_ERS_RESULT_DISCONNECT;
8752         }
8753
8754         err = pci_cleanup_aer_uncorrect_error_status(pdev);
8755         if (err) {
8756                 dev_info(&pdev->dev,
8757                          "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
8758                          err);
8759                 /* non-fatal, continue */
8760         }
8761
8762         return result;
8763 }
8764
8765 /**
8766  * i40e_pci_error_resume - restart operations after PCI error recovery
8767  * @pdev: PCI device information struct
8768  *
8769  * Called to allow the driver to bring things back up after PCI error
8770  * and/or reset recovery has finished.
8771  **/
8772 static void i40e_pci_error_resume(struct pci_dev *pdev)
8773 {
8774         struct i40e_pf *pf = pci_get_drvdata(pdev);
8775
8776         dev_info(&pdev->dev, "%s\n", __func__);
8777         if (test_bit(__I40E_SUSPENDED, &pf->state))
8778                 return;
8779
8780         rtnl_lock();
8781         i40e_handle_reset_warning(pf);
8782         rtnl_lock();
8783 }
8784
8785 /**
8786  * i40e_shutdown - PCI callback for shutting down
8787  * @pdev: PCI device information struct
8788  **/
8789 static void i40e_shutdown(struct pci_dev *pdev)
8790 {
8791         struct i40e_pf *pf = pci_get_drvdata(pdev);
8792         struct i40e_hw *hw = &pf->hw;
8793
8794         set_bit(__I40E_SUSPENDED, &pf->state);
8795         set_bit(__I40E_DOWN, &pf->state);
8796         rtnl_lock();
8797         i40e_prep_for_reset(pf);
8798         rtnl_unlock();
8799
8800         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
8801         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
8802
8803         if (system_state == SYSTEM_POWER_OFF) {
8804                 pci_wake_from_d3(pdev, pf->wol_en);
8805                 pci_set_power_state(pdev, PCI_D3hot);
8806         }
8807 }
8808
8809 #ifdef CONFIG_PM
8810 /**
8811  * i40e_suspend - PCI callback for moving to D3
8812  * @pdev: PCI device information struct
8813  **/
8814 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
8815 {
8816         struct i40e_pf *pf = pci_get_drvdata(pdev);
8817         struct i40e_hw *hw = &pf->hw;
8818
8819         set_bit(__I40E_SUSPENDED, &pf->state);
8820         set_bit(__I40E_DOWN, &pf->state);
8821         rtnl_lock();
8822         i40e_prep_for_reset(pf);
8823         rtnl_unlock();
8824
8825         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
8826         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
8827
8828         pci_wake_from_d3(pdev, pf->wol_en);
8829         pci_set_power_state(pdev, PCI_D3hot);
8830
8831         return 0;
8832 }
8833
8834 /**
8835  * i40e_resume - PCI callback for waking up from D3
8836  * @pdev: PCI device information struct
8837  **/
8838 static int i40e_resume(struct pci_dev *pdev)
8839 {
8840         struct i40e_pf *pf = pci_get_drvdata(pdev);
8841         u32 err;
8842
8843         pci_set_power_state(pdev, PCI_D0);
8844         pci_restore_state(pdev);
8845         /* pci_restore_state() clears dev->state_saves, so
8846          * call pci_save_state() again to restore it.
8847          */
8848         pci_save_state(pdev);
8849
8850         err = pci_enable_device_mem(pdev);
8851         if (err) {
8852                 dev_err(&pdev->dev,
8853                         "%s: Cannot enable PCI device from suspend\n",
8854                         __func__);
8855                 return err;
8856         }
8857         pci_set_master(pdev);
8858
8859         /* no wakeup events while running */
8860         pci_wake_from_d3(pdev, false);
8861
8862         /* handling the reset will rebuild the device state */
8863         if (test_and_clear_bit(__I40E_SUSPENDED, &pf->state)) {
8864                 clear_bit(__I40E_DOWN, &pf->state);
8865                 rtnl_lock();
8866                 i40e_reset_and_rebuild(pf, false);
8867                 rtnl_unlock();
8868         }
8869
8870         return 0;
8871 }
8872
8873 #endif
8874 static const struct pci_error_handlers i40e_err_handler = {
8875         .error_detected = i40e_pci_error_detected,
8876         .slot_reset = i40e_pci_error_slot_reset,
8877         .resume = i40e_pci_error_resume,
8878 };
8879
8880 static struct pci_driver i40e_driver = {
8881         .name     = i40e_driver_name,
8882         .id_table = i40e_pci_tbl,
8883         .probe    = i40e_probe,
8884         .remove   = i40e_remove,
8885 #ifdef CONFIG_PM
8886         .suspend  = i40e_suspend,
8887         .resume   = i40e_resume,
8888 #endif
8889         .shutdown = i40e_shutdown,
8890         .err_handler = &i40e_err_handler,
8891         .sriov_configure = i40e_pci_sriov_configure,
8892 };
8893
8894 /**
8895  * i40e_init_module - Driver registration routine
8896  *
8897  * i40e_init_module is the first routine called when the driver is
8898  * loaded. All it does is register with the PCI subsystem.
8899  **/
8900 static int __init i40e_init_module(void)
8901 {
8902         pr_info("%s: %s - version %s\n", i40e_driver_name,
8903                 i40e_driver_string, i40e_driver_version_str);
8904         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
8905         i40e_dbg_init();
8906         return pci_register_driver(&i40e_driver);
8907 }
8908 module_init(i40e_init_module);
8909
8910 /**
8911  * i40e_exit_module - Driver exit cleanup routine
8912  *
8913  * i40e_exit_module is called just before the driver is removed
8914  * from memory.
8915  **/
8916 static void __exit i40e_exit_module(void)
8917 {
8918         pci_unregister_driver(&i40e_driver);
8919         i40e_dbg_exit();
8920 }
8921 module_exit(i40e_exit_module);