drivers/net/ethernet/intel/i40e/i40e_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2017 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 #include <linux/etherdevice.h>
  28 #include <linux/of_net.h>
  29 #include <linux/pci.h>
  30 #include <linux/bpf.h>
  31
  32 /* Local includes */
  33 #include "i40e.h"
  34 #include "i40e_diag.h"
  35 #include <net/udp_tunnel.h>
  36 /* All i40e tracepoints are defined by the include below, which
  37  * must be included exactly once across the whole kernel with
  38  * CREATE_TRACE_POINTS defined
  39  */
  40 #define CREATE_TRACE_POINTS
  41 #include "i40e_trace.h"
  42
  43 const char i40e_driver_name[] = "i40e";
  44 static const char i40e_driver_string[] =
  45                         "Intel(R) Ethernet Connection XL710 Network Driver";
  46
  47 #define DRV_KERN "-k"
  48
  49 #define DRV_VERSION_MAJOR 2
  50 #define DRV_VERSION_MINOR 1
  51 #define DRV_VERSION_BUILD 14
  52 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  53              __stringify(DRV_VERSION_MINOR) "." \
  54              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  55 const char i40e_driver_version_str[] = DRV_VERSION;
  56 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation.";
  57
  58 /* a bit of forward declarations */
  59 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  60 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
  61 static int i40e_add_vsi(struct i40e_vsi *vsi);
  62 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  63 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  64 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  65 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  66 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  67 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired);
  68 static int i40e_reset(struct i40e_pf *pf);
  69 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  70 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  71 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  72
  73 /* i40e_pci_tbl - PCI Device ID Table
  74  *
  75  * Last entry must be all 0s
  76  *
  77  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  78  *   Class, Class Mask, private data (not used) }
  79  */
  80 static const struct pci_device_id i40e_pci_tbl[] = {
  81         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  82         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  83         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  84         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  85         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  86         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  87         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  88         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  89         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
  90         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
  91         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
  92         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
  93         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
  94         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
  95         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
  96         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  97         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
  98         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
  99         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
 100         /* required last entry */
 101         {0, }
 102 };
 103 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
 104
 105 #define I40E_MAX_VF_COUNT 128
 106 static int debug = -1;
 107 module_param(debug, uint, 0);
 108 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
 109
 110 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 111 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
 112 MODULE_LICENSE("GPL");
 113 MODULE_VERSION(DRV_VERSION);
 114
 115 static struct workqueue_struct *i40e_wq;
 116
 117 /**
 118  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
 119  * @hw:   pointer to the HW structure
 120  * @mem:  ptr to mem struct to fill out
 121  * @size: size of memory requested
 122  * @alignment: what to align the allocation to
 123  **/
 124 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 125                             u64 size, u32 alignment)
 126 {
 127         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 128
 129         mem->size = ALIGN(size, alignment);
 130         mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
 131                                       &mem->pa, GFP_KERNEL);
 132         if (!mem->va)
 133                 return -ENOMEM;
 134
 135         return 0;
 136 }
 137
 138 /**
 139  * i40e_free_dma_mem_d - OS specific memory free for shared code
 140  * @hw:   pointer to the HW structure
 141  * @mem:  ptr to mem struct to free
 142  **/
 143 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 144 {
 145         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 146
 147         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 148         mem->va = NULL;
 149         mem->pa = 0;
 150         mem->size = 0;
 151
 152         return 0;
 153 }
 154
 155 /**
 156  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 157  * @hw:   pointer to the HW structure
 158  * @mem:  ptr to mem struct to fill out
 159  * @size: size of memory requested
 160  **/
 161 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 162                              u32 size)
 163 {
 164         mem->size = size;
 165         mem->va = kzalloc(size, GFP_KERNEL);
 166
 167         if (!mem->va)
 168                 return -ENOMEM;
 169
 170         return 0;
 171 }
 172
 173 /**
 174  * i40e_free_virt_mem_d - OS specific memory free for shared code
 175  * @hw:   pointer to the HW structure
 176  * @mem:  ptr to mem struct to free
 177  **/
 178 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 179 {
 180         /* it's ok to kfree a NULL pointer */
 181         kfree(mem->va);
 182         mem->va = NULL;
 183         mem->size = 0;
 184
 185         return 0;
 186 }
 187
 188 /**
 189  * i40e_get_lump - find a lump of free generic resource
 190  * @pf: board private structure
 191  * @pile: the pile of resource to search
 192  * @needed: the number of items needed
 193  * @id: an owner id to stick on the items assigned
 194  *
 195  * Returns the base item index of the lump, or negative for error
 196  *
 197  * The search_hint trick and lack of advanced fit-finding only work
 198  * because we're highly likely to have all the same size lump requests.
 199  * Linear search time and any fragmentation should be minimal.
 200  **/
 201 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 202                          u16 needed, u16 id)
 203 {
 204         int ret = -ENOMEM;
 205         int i, j;
 206
 207         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 208                 dev_info(&pf->pdev->dev,
 209                          "param err: pile=%p needed=%d id=0x%04x\n",
 210                          pile, needed, id);
 211                 return -EINVAL;
 212         }
 213
 214         /* start the linear search with an imperfect hint */
 215         i = pile->search_hint;
 216         while (i < pile->num_entries) {
 217                 /* skip already allocated entries */
 218                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 219                         i++;
 220                         continue;
 221                 }
 222
 223                 /* do we have enough in this lump? */
 224                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 225                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 226                                 break;
 227                 }
 228
 229                 if (j == needed) {
 230                         /* there was enough, so assign it to the requestor */
 231                         for (j = 0; j < needed; j++)
 232                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 233                         ret = i;
 234                         pile->search_hint = i + j;
 235                         break;
 236                 }
 237
 238                 /* not enough, so skip over it and continue looking */
 239                 i += j;
 240         }
 241
 242         return ret;
 243 }
 244
 245 /**
 246  * i40e_put_lump - return a lump of generic resource
 247  * @pile: the pile of resource to search
 248  * @index: the base item index
 249  * @id: the owner id of the items assigned
 250  *
 251  * Returns the count of items in the lump
 252  **/
 253 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 254 {
 255         int valid_id = (id | I40E_PILE_VALID_BIT);
 256         int count = 0;
 257         int i;
 258
 259         if (!pile || index >= pile->num_entries)
 260                 return -EINVAL;
 261
 262         for (i = index;
 263              i < pile->num_entries && pile->list[i] == valid_id;
 264              i++) {
 265                 pile->list[i] = 0;
 266                 count++;
 267         }
 268
 269         if (count && index < pile->search_hint)
 270                 pile->search_hint = index;
 271
 272         return count;
 273 }
 274
 275 /**
 276  * i40e_find_vsi_from_id - searches for the vsi with the given id
 277  * @pf - the pf structure to search for the vsi
 278  * @id - id of the vsi it is searching for
 279  **/
 280 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 281 {
 282         int i;
 283
 284         for (i = 0; i < pf->num_alloc_vsi; i++)
 285                 if (pf->vsi[i] && (pf->vsi[i]->id == id))
 286                         return pf->vsi[i];
 287
 288         return NULL;
 289 }
 290
 291 /**
 292  * i40e_service_event_schedule - Schedule the service task to wake up
 293  * @pf: board private structure
 294  *
 295  * If not already scheduled, this puts the task into the work queue
 296  **/
 297 void i40e_service_event_schedule(struct i40e_pf *pf)
 298 {
 299         if (!test_bit(__I40E_DOWN, pf->state) &&
 300             !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
 301                 queue_work(i40e_wq, &pf->service_task);
 302 }
 303
 304 /**
 305  * i40e_tx_timeout - Respond to a Tx Hang
 306  * @netdev: network interface device structure
 307  *
 308  * If any port has noticed a Tx timeout, it is likely that the whole
 309  * device is munged, not just the one netdev port, so go for the full
 310  * reset.
 311  **/
 312 static void i40e_tx_timeout(struct net_device *netdev)
 313 {
 314         struct i40e_netdev_priv *np = netdev_priv(netdev);
 315         struct i40e_vsi *vsi = np->vsi;
 316         struct i40e_pf *pf = vsi->back;
 317         struct i40e_ring *tx_ring = NULL;
 318         unsigned int i, hung_queue = 0;
 319         u32 head, val;
 320
 321         pf->tx_timeout_count++;
 322
 323         /* find the stopped queue the same way the stack does */
 324         for (i = 0; i < netdev->num_tx_queues; i++) {
 325                 struct netdev_queue *q;
 326                 unsigned long trans_start;
 327
 328                 q = netdev_get_tx_queue(netdev, i);
 329                 trans_start = q->trans_start;
 330                 if (netif_xmit_stopped(q) &&
 331                     time_after(jiffies,
 332                                (trans_start + netdev->watchdog_timeo))) {
 333                         hung_queue = i;
 334                         break;
 335                 }
 336         }
 337
 338         if (i == netdev->num_tx_queues) {
 339                 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n");
 340         } else {
 341                 /* now that we have an index, find the tx_ring struct */
 342                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 343                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
 344                                 if (hung_queue ==
 345                                     vsi->tx_rings[i]->queue_index) {
 346                                         tx_ring = vsi->tx_rings[i];
 347                                         break;
 348                                 }
 349                         }
 350                 }
 351         }
 352
 353         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 354                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
 355         else if (time_before(jiffies,
 356                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
 357                 return;   /* don't do any new action before the next timeout */
 358
 359         if (tx_ring) {
 360                 head = i40e_get_head(tx_ring);
 361                 /* Read interrupt register */
 362                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
 363                         val = rd32(&pf->hw,
 364                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 365                                                 tx_ring->vsi->base_vector - 1));
 366                 else
 367                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
 368
 369                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
 370                             vsi->seid, hung_queue, tx_ring->next_to_clean,
 371                             head, tx_ring->next_to_use,
 372                             readl(tx_ring->tail), val);
 373         }
 374
 375         pf->tx_timeout_last_recovery = jiffies;
 376         netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n",
 377                     pf->tx_timeout_recovery_level, hung_queue);
 378
 379         switch (pf->tx_timeout_recovery_level) {
 380         case 1:
 381                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
 382                 break;
 383         case 2:
 384                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
 385                 break;
 386         case 3:
 387                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
 388                 break;
 389         default:
 390                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 391                 break;
 392         }
 393
 394         i40e_service_event_schedule(pf);
 395         pf->tx_timeout_recovery_level++;
 396 }
 397
 398 /**
 399  * i40e_get_vsi_stats_struct - Get System Network Statistics
 400  * @vsi: the VSI we care about
 401  *
 402  * Returns the address of the device statistics structure.
 403  * The statistics are actually updated from the service task.
 404  **/
 405 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 406 {
 407         return &vsi->net_stats;
 408 }
 409
 410 /**
 411  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
 412  * @ring: Tx ring to get statistics from
 413  * @stats: statistics entry to be updated
 414  **/
 415 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
 416                                             struct rtnl_link_stats64 *stats)
 417 {
 418         u64 bytes, packets;
 419         unsigned int start;
 420
 421         do {
 422                 start = u64_stats_fetch_begin_irq(&ring->syncp);
 423                 packets = ring->stats.packets;
 424                 bytes   = ring->stats.bytes;
 425         } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 426
 427         stats->tx_packets += packets;
 428         stats->tx_bytes   += bytes;
 429 }
 430
 431 /**
 432  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 433  * @netdev: network interface device structure
 434  *
 435  * Returns the address of the device statistics structure.
 436  * The statistics are actually updated from the service task.
 437  **/
 438 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
 439                                   struct rtnl_link_stats64 *stats)
 440 {
 441         struct i40e_netdev_priv *np = netdev_priv(netdev);
 442         struct i40e_ring *tx_ring, *rx_ring;
 443         struct i40e_vsi *vsi = np->vsi;
 444         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 445         int i;
 446
 447         if (test_bit(__I40E_VSI_DOWN, vsi->state))
 448                 return;
 449
 450         if (!vsi->tx_rings)
 451                 return;
 452
 453         rcu_read_lock();
 454         for (i = 0; i < vsi->num_queue_pairs; i++) {
 455                 u64 bytes, packets;
 456                 unsigned int start;
 457
 458                 tx_ring = READ_ONCE(vsi->tx_rings[i]);
 459                 if (!tx_ring)
 460                         continue;
 461                 i40e_get_netdev_stats_struct_tx(tx_ring, stats);
 462
 463                 rx_ring = &tx_ring[1];
 464
 465                 do {
 466                         start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
 467                         packets = rx_ring->stats.packets;
 468                         bytes   = rx_ring->stats.bytes;
 469                 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
 470
 471                 stats->rx_packets += packets;
 472                 stats->rx_bytes   += bytes;
 473
 474                 if (i40e_enabled_xdp_vsi(vsi))
 475                         i40e_get_netdev_stats_struct_tx(&rx_ring[1], stats);
 476         }
 477         rcu_read_unlock();
 478
 479         /* following stats updated by i40e_watchdog_subtask() */
 480         stats->multicast        = vsi_stats->multicast;
 481         stats->tx_errors        = vsi_stats->tx_errors;
 482         stats->tx_dropped       = vsi_stats->tx_dropped;
 483         stats->rx_errors        = vsi_stats->rx_errors;
 484         stats->rx_dropped       = vsi_stats->rx_dropped;
 485         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 486         stats->rx_length_errors = vsi_stats->rx_length_errors;
 487 }
 488
 489 /**
 490  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 491  * @vsi: the VSI to have its stats reset
 492  **/
 493 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 494 {
 495         struct rtnl_link_stats64 *ns;
 496         int i;
 497
 498         if (!vsi)
 499                 return;
 500
 501         ns = i40e_get_vsi_stats_struct(vsi);
 502         memset(ns, 0, sizeof(*ns));
 503         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 504         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 505         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 506         if (vsi->rx_rings && vsi->rx_rings[0]) {
 507                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 508                         memset(&vsi->rx_rings[i]->stats, 0,
 509                                sizeof(vsi->rx_rings[i]->stats));
 510                         memset(&vsi->rx_rings[i]->rx_stats, 0,
 511                                sizeof(vsi->rx_rings[i]->rx_stats));
 512                         memset(&vsi->tx_rings[i]->stats, 0,
 513                                sizeof(vsi->tx_rings[i]->stats));
 514                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 515                                sizeof(vsi->tx_rings[i]->tx_stats));
 516                 }
 517         }
 518         vsi->stat_offsets_loaded = false;
 519 }
 520
 521 /**
 522  * i40e_pf_reset_stats - Reset all of the stats for the given PF
 523  * @pf: the PF to be reset
 524  **/
 525 void i40e_pf_reset_stats(struct i40e_pf *pf)
 526 {
 527         int i;
 528
 529         memset(&pf->stats, 0, sizeof(pf->stats));
 530         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 531         pf->stat_offsets_loaded = false;
 532
 533         for (i = 0; i < I40E_MAX_VEB; i++) {
 534                 if (pf->veb[i]) {
 535                         memset(&pf->veb[i]->stats, 0,
 536                                sizeof(pf->veb[i]->stats));
 537                         memset(&pf->veb[i]->stats_offsets, 0,
 538                                sizeof(pf->veb[i]->stats_offsets));
 539                         pf->veb[i]->stat_offsets_loaded = false;
 540                 }
 541         }
 542         pf->hw_csum_rx_error = 0;
 543 }
 544
 545 /**
 546  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 547  * @hw: ptr to the hardware info
 548  * @hireg: the high 32 bit reg to read
 549  * @loreg: the low 32 bit reg to read
 550  * @offset_loaded: has the initial offset been loaded yet
 551  * @offset: ptr to current offset value
 552  * @stat: ptr to the stat
 553  *
 554  * Since the device stats are not reset at PFReset, they likely will not
 555  * be zeroed when the driver starts.  We'll save the first values read
 556  * and use them as offsets to be subtracted from the raw values in order
 557  * to report stats that count from zero.  In the process, we also manage
 558  * the potential roll-over.
 559  **/
 560 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 561                                bool offset_loaded, u64 *offset, u64 *stat)
 562 {
 563         u64 new_data;
 564
 565         if (hw->device_id == I40E_DEV_ID_QEMU) {
 566                 new_data = rd32(hw, loreg);
 567                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 568         } else {
 569                 new_data = rd64(hw, loreg);
 570         }
 571         if (!offset_loaded)
 572                 *offset = new_data;
 573         if (likely(new_data >= *offset))
 574                 *stat = new_data - *offset;
 575         else
 576                 *stat = (new_data + BIT_ULL(48)) - *offset;
 577         *stat &= 0xFFFFFFFFFFFFULL;
 578 }
 579
 580 /**
 581  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 582  * @hw: ptr to the hardware info
 583  * @reg: the hw reg to read
 584  * @offset_loaded: has the initial offset been loaded yet
 585  * @offset: ptr to current offset value
 586  * @stat: ptr to the stat
 587  **/
 588 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 589                                bool offset_loaded, u64 *offset, u64 *stat)
 590 {
 591         u32 new_data;
 592
 593         new_data = rd32(hw, reg);
 594         if (!offset_loaded)
 595                 *offset = new_data;
 596         if (likely(new_data >= *offset))
 597                 *stat = (u32)(new_data - *offset);
 598         else
 599                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
 600 }
 601
 602 /**
 603  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 604  * @vsi: the VSI to be updated
 605  **/
 606 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 607 {
 608         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 609         struct i40e_pf *pf = vsi->back;
 610         struct i40e_hw *hw = &pf->hw;
 611         struct i40e_eth_stats *oes;
 612         struct i40e_eth_stats *es;     /* device's eth stats */
 613
 614         es = &vsi->eth_stats;
 615         oes = &vsi->eth_stats_offsets;
 616
 617         /* Gather up the stats that the hw collects */
 618         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 619                            vsi->stat_offsets_loaded,
 620                            &oes->tx_errors, &es->tx_errors);
 621         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 622                            vsi->stat_offsets_loaded,
 623                            &oes->rx_discards, &es->rx_discards);
 624         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 625                            vsi->stat_offsets_loaded,
 626                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 627         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 628                            vsi->stat_offsets_loaded,
 629                            &oes->tx_errors, &es->tx_errors);
 630
 631         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 632                            I40E_GLV_GORCL(stat_idx),
 633                            vsi->stat_offsets_loaded,
 634                            &oes->rx_bytes, &es->rx_bytes);
 635         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 636                            I40E_GLV_UPRCL(stat_idx),
 637                            vsi->stat_offsets_loaded,
 638                            &oes->rx_unicast, &es->rx_unicast);
 639         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 640                            I40E_GLV_MPRCL(stat_idx),
 641                            vsi->stat_offsets_loaded,
 642                            &oes->rx_multicast, &es->rx_multicast);
 643         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 644                            I40E_GLV_BPRCL(stat_idx),
 645                            vsi->stat_offsets_loaded,
 646                            &oes->rx_broadcast, &es->rx_broadcast);
 647
 648         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 649                            I40E_GLV_GOTCL(stat_idx),
 650                            vsi->stat_offsets_loaded,
 651                            &oes->tx_bytes, &es->tx_bytes);
 652         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 653                            I40E_GLV_UPTCL(stat_idx),
 654                            vsi->stat_offsets_loaded,
 655                            &oes->tx_unicast, &es->tx_unicast);
 656         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 657                            I40E_GLV_MPTCL(stat_idx),
 658                            vsi->stat_offsets_loaded,
 659                            &oes->tx_multicast, &es->tx_multicast);
 660         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 661                            I40E_GLV_BPTCL(stat_idx),
 662                            vsi->stat_offsets_loaded,
 663                            &oes->tx_broadcast, &es->tx_broadcast);
 664         vsi->stat_offsets_loaded = true;
 665 }
 666
 667 /**
 668  * i40e_update_veb_stats - Update Switch component statistics
 669  * @veb: the VEB being updated
 670  **/
 671 static void i40e_update_veb_stats(struct i40e_veb *veb)
 672 {
 673         struct i40e_pf *pf = veb->pf;
 674         struct i40e_hw *hw = &pf->hw;
 675         struct i40e_eth_stats *oes;
 676         struct i40e_eth_stats *es;     /* device's eth stats */
 677         struct i40e_veb_tc_stats *veb_oes;
 678         struct i40e_veb_tc_stats *veb_es;
 679         int i, idx = 0;
 680
 681         idx = veb->stats_idx;
 682         es = &veb->stats;
 683         oes = &veb->stats_offsets;
 684         veb_es = &veb->tc_stats;
 685         veb_oes = &veb->tc_stats_offsets;
 686
 687         /* Gather up the stats that the hw collects */
 688         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 689                            veb->stat_offsets_loaded,
 690                            &oes->tx_discards, &es->tx_discards);
 691         if (hw->revision_id > 0)
 692                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 693                                    veb->stat_offsets_loaded,
 694                                    &oes->rx_unknown_protocol,
 695                                    &es->rx_unknown_protocol);
 696         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 697                            veb->stat_offsets_loaded,
 698                            &oes->rx_bytes, &es->rx_bytes);
 699         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 700                            veb->stat_offsets_loaded,
 701                            &oes->rx_unicast, &es->rx_unicast);
 702         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 703                            veb->stat_offsets_loaded,
 704                            &oes->rx_multicast, &es->rx_multicast);
 705         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 706                            veb->stat_offsets_loaded,
 707                            &oes->rx_broadcast, &es->rx_broadcast);
 708
 709         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 710                            veb->stat_offsets_loaded,
 711                            &oes->tx_bytes, &es->tx_bytes);
 712         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 713                            veb->stat_offsets_loaded,
 714                            &oes->tx_unicast, &es->tx_unicast);
 715         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 716                            veb->stat_offsets_loaded,
 717                            &oes->tx_multicast, &es->tx_multicast);
 718         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 719                            veb->stat_offsets_loaded,
 720                            &oes->tx_broadcast, &es->tx_broadcast);
 721         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 722                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
 723                                    I40E_GLVEBTC_RPCL(i, idx),
 724                                    veb->stat_offsets_loaded,
 725                                    &veb_oes->tc_rx_packets[i],
 726                                    &veb_es->tc_rx_packets[i]);
 727                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
 728                                    I40E_GLVEBTC_RBCL(i, idx),
 729                                    veb->stat_offsets_loaded,
 730                                    &veb_oes->tc_rx_bytes[i],
 731                                    &veb_es->tc_rx_bytes[i]);
 732                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
 733                                    I40E_GLVEBTC_TPCL(i, idx),
 734                                    veb->stat_offsets_loaded,
 735                                    &veb_oes->tc_tx_packets[i],
 736                                    &veb_es->tc_tx_packets[i]);
 737                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
 738                                    I40E_GLVEBTC_TBCL(i, idx),
 739                                    veb->stat_offsets_loaded,
 740                                    &veb_oes->tc_tx_bytes[i],
 741                                    &veb_es->tc_tx_bytes[i]);
 742         }
 743         veb->stat_offsets_loaded = true;
 744 }
 745
 746 /**
 747  * i40e_update_vsi_stats - Update the vsi statistics counters.
 748  * @vsi: the VSI to be updated
 749  *
 750  * There are a few instances where we store the same stat in a
 751  * couple of different structs.  This is partly because we have
 752  * the netdev stats that need to be filled out, which is slightly
 753  * different from the "eth_stats" defined by the chip and used in
 754  * VF communications.  We sort it out here.
 755  **/
 756 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 757 {
 758         struct i40e_pf *pf = vsi->back;
 759         struct rtnl_link_stats64 *ons;
 760         struct rtnl_link_stats64 *ns;   /* netdev stats */
 761         struct i40e_eth_stats *oes;
 762         struct i40e_eth_stats *es;     /* device's eth stats */
 763         u32 tx_restart, tx_busy;
 764         struct i40e_ring *p;
 765         u32 rx_page, rx_buf;
 766         u64 bytes, packets;
 767         unsigned int start;
 768         u64 tx_linearize;
 769         u64 tx_force_wb;
 770         u64 rx_p, rx_b;
 771         u64 tx_p, tx_b;
 772         u16 q;
 773
 774         if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
 775             test_bit(__I40E_CONFIG_BUSY, pf->state))
 776                 return;
 777
 778         ns = i40e_get_vsi_stats_struct(vsi);
 779         ons = &vsi->net_stats_offsets;
 780         es = &vsi->eth_stats;
 781         oes = &vsi->eth_stats_offsets;
 782
 783         /* Gather up the netdev and vsi stats that the driver collects
 784          * on the fly during packet processing
 785          */
 786         rx_b = rx_p = 0;
 787         tx_b = tx_p = 0;
 788         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
 789         rx_page = 0;
 790         rx_buf = 0;
 791         rcu_read_lock();
 792         for (q = 0; q < vsi->num_queue_pairs; q++) {
 793                 /* locate Tx ring */
 794                 p = READ_ONCE(vsi->tx_rings[q]);
 795
 796                 do {
 797                         start = u64_stats_fetch_begin_irq(&p->syncp);
 798                         packets = p->stats.packets;
 799                         bytes = p->stats.bytes;
 800                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 801                 tx_b += bytes;
 802                 tx_p += packets;
 803                 tx_restart += p->tx_stats.restart_queue;
 804                 tx_busy += p->tx_stats.tx_busy;
 805                 tx_linearize += p->tx_stats.tx_linearize;
 806                 tx_force_wb += p->tx_stats.tx_force_wb;
 807
 808                 /* Rx queue is part of the same block as Tx queue */
 809                 p = &p[1];
 810                 do {
 811                         start = u64_stats_fetch_begin_irq(&p->syncp);
 812                         packets = p->stats.packets;
 813                         bytes = p->stats.bytes;
 814                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 815                 rx_b += bytes;
 816                 rx_p += packets;
 817                 rx_buf += p->rx_stats.alloc_buff_failed;
 818                 rx_page += p->rx_stats.alloc_page_failed;
 819         }
 820         rcu_read_unlock();
 821         vsi->tx_restart = tx_restart;
 822         vsi->tx_busy = tx_busy;
 823         vsi->tx_linearize = tx_linearize;
 824         vsi->tx_force_wb = tx_force_wb;
 825         vsi->rx_page_failed = rx_page;
 826         vsi->rx_buf_failed = rx_buf;
 827
 828         ns->rx_packets = rx_p;
 829         ns->rx_bytes = rx_b;
 830         ns->tx_packets = tx_p;
 831         ns->tx_bytes = tx_b;
 832
 833         /* update netdev stats from eth stats */
 834         i40e_update_eth_stats(vsi);
 835         ons->tx_errors = oes->tx_errors;
 836         ns->tx_errors = es->tx_errors;
 837         ons->multicast = oes->rx_multicast;
 838         ns->multicast = es->rx_multicast;
 839         ons->rx_dropped = oes->rx_discards;
 840         ns->rx_dropped = es->rx_discards;
 841         ons->tx_dropped = oes->tx_discards;
 842         ns->tx_dropped = es->tx_discards;
 843
 844         /* pull in a couple PF stats if this is the main vsi */
 845         if (vsi == pf->vsi[pf->lan_vsi]) {
 846                 ns->rx_crc_errors = pf->stats.crc_errors;
 847                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 848                 ns->rx_length_errors = pf->stats.rx_length_errors;
 849         }
 850 }
 851
 852 /**
 853  * i40e_update_pf_stats - Update the PF statistics counters.
 854  * @pf: the PF to be updated
 855  **/
 856 static void i40e_update_pf_stats(struct i40e_pf *pf)
 857 {
 858         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 859         struct i40e_hw_port_stats *nsd = &pf->stats;
 860         struct i40e_hw *hw = &pf->hw;
 861         u32 val;
 862         int i;
 863
 864         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 865                            I40E_GLPRT_GORCL(hw->port),
 866                            pf->stat_offsets_loaded,
 867                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 868         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 869                            I40E_GLPRT_GOTCL(hw->port),
 870                            pf->stat_offsets_loaded,
 871                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 872         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 873                            pf->stat_offsets_loaded,
 874                            &osd->eth.rx_discards,
 875                            &nsd->eth.rx_discards);
 876         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 877                            I40E_GLPRT_UPRCL(hw->port),
 878                            pf->stat_offsets_loaded,
 879                            &osd->eth.rx_unicast,
 880                            &nsd->eth.rx_unicast);
 881         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 882                            I40E_GLPRT_MPRCL(hw->port),
 883                            pf->stat_offsets_loaded,
 884                            &osd->eth.rx_multicast,
 885                            &nsd->eth.rx_multicast);
 886         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 887                            I40E_GLPRT_BPRCL(hw->port),
 888                            pf->stat_offsets_loaded,
 889                            &osd->eth.rx_broadcast,
 890                            &nsd->eth.rx_broadcast);
 891         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 892                            I40E_GLPRT_UPTCL(hw->port),
 893                            pf->stat_offsets_loaded,
 894                            &osd->eth.tx_unicast,
 895                            &nsd->eth.tx_unicast);
 896         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 897                            I40E_GLPRT_MPTCL(hw->port),
 898                            pf->stat_offsets_loaded,
 899                            &osd->eth.tx_multicast,
 900                            &nsd->eth.tx_multicast);
 901         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 902                            I40E_GLPRT_BPTCL(hw->port),
 903                            pf->stat_offsets_loaded,
 904                            &osd->eth.tx_broadcast,
 905                            &nsd->eth.tx_broadcast);
 906
 907         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 908                            pf->stat_offsets_loaded,
 909                            &osd->tx_dropped_link_down,
 910                            &nsd->tx_dropped_link_down);
 911
 912         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 913                            pf->stat_offsets_loaded,
 914                            &osd->crc_errors, &nsd->crc_errors);
 915
 916         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 917                            pf->stat_offsets_loaded,
 918                            &osd->illegal_bytes, &nsd->illegal_bytes);
 919
 920         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 921                            pf->stat_offsets_loaded,
 922                            &osd->mac_local_faults,
 923                            &nsd->mac_local_faults);
 924         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 925                            pf->stat_offsets_loaded,
 926                            &osd->mac_remote_faults,
 927                            &nsd->mac_remote_faults);
 928
 929         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 930                            pf->stat_offsets_loaded,
 931                            &osd->rx_length_errors,
 932                            &nsd->rx_length_errors);
 933
 934         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 935                            pf->stat_offsets_loaded,
 936                            &osd->link_xon_rx, &nsd->link_xon_rx);
 937         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 938                            pf->stat_offsets_loaded,
 939                            &osd->link_xon_tx, &nsd->link_xon_tx);
 940         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 941                            pf->stat_offsets_loaded,
 942                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 943         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 944                            pf->stat_offsets_loaded,
 945                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
 946
 947         for (i = 0; i < 8; i++) {
 948                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 949                                    pf->stat_offsets_loaded,
 950                                    &osd->priority_xoff_rx[i],
 951                                    &nsd->priority_xoff_rx[i]);
 952                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 953                                    pf->stat_offsets_loaded,
 954                                    &osd->priority_xon_rx[i],
 955                                    &nsd->priority_xon_rx[i]);
 956                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 957                                    pf->stat_offsets_loaded,
 958                                    &osd->priority_xon_tx[i],
 959                                    &nsd->priority_xon_tx[i]);
 960                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 961                                    pf->stat_offsets_loaded,
 962                                    &osd->priority_xoff_tx[i],
 963                                    &nsd->priority_xoff_tx[i]);
 964                 i40e_stat_update32(hw,
 965                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
 966                                    pf->stat_offsets_loaded,
 967                                    &osd->priority_xon_2_xoff[i],
 968                                    &nsd->priority_xon_2_xoff[i]);
 969         }
 970
 971         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
 972                            I40E_GLPRT_PRC64L(hw->port),
 973                            pf->stat_offsets_loaded,
 974                            &osd->rx_size_64, &nsd->rx_size_64);
 975         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
 976                            I40E_GLPRT_PRC127L(hw->port),
 977                            pf->stat_offsets_loaded,
 978                            &osd->rx_size_127, &nsd->rx_size_127);
 979         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
 980                            I40E_GLPRT_PRC255L(hw->port),
 981                            pf->stat_offsets_loaded,
 982                            &osd->rx_size_255, &nsd->rx_size_255);
 983         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
 984                            I40E_GLPRT_PRC511L(hw->port),
 985                            pf->stat_offsets_loaded,
 986                            &osd->rx_size_511, &nsd->rx_size_511);
 987         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
 988                            I40E_GLPRT_PRC1023L(hw->port),
 989                            pf->stat_offsets_loaded,
 990                            &osd->rx_size_1023, &nsd->rx_size_1023);
 991         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
 992                            I40E_GLPRT_PRC1522L(hw->port),
 993                            pf->stat_offsets_loaded,
 994                            &osd->rx_size_1522, &nsd->rx_size_1522);
 995         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
 996                            I40E_GLPRT_PRC9522L(hw->port),
 997                            pf->stat_offsets_loaded,
 998                            &osd->rx_size_big, &nsd->rx_size_big);
 999
1000         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1001                            I40E_GLPRT_PTC64L(hw->port),
1002                            pf->stat_offsets_loaded,
1003                            &osd->tx_size_64, &nsd->tx_size_64);
1004         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1005                            I40E_GLPRT_PTC127L(hw->port),
1006                            pf->stat_offsets_loaded,
1007                            &osd->tx_size_127, &nsd->tx_size_127);
1008         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1009                            I40E_GLPRT_PTC255L(hw->port),
1010                            pf->stat_offsets_loaded,
1011                            &osd->tx_size_255, &nsd->tx_size_255);
1012         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1013                            I40E_GLPRT_PTC511L(hw->port),
1014                            pf->stat_offsets_loaded,
1015                            &osd->tx_size_511, &nsd->tx_size_511);
1016         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1017                            I40E_GLPRT_PTC1023L(hw->port),
1018                            pf->stat_offsets_loaded,
1019                            &osd->tx_size_1023, &nsd->tx_size_1023);
1020         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1021                            I40E_GLPRT_PTC1522L(hw->port),
1022                            pf->stat_offsets_loaded,
1023                            &osd->tx_size_1522, &nsd->tx_size_1522);
1024         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1025                            I40E_GLPRT_PTC9522L(hw->port),
1026                            pf->stat_offsets_loaded,
1027                            &osd->tx_size_big, &nsd->tx_size_big);
1028
1029         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1030                            pf->stat_offsets_loaded,
1031                            &osd->rx_undersize, &nsd->rx_undersize);
1032         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1033                            pf->stat_offsets_loaded,
1034                            &osd->rx_fragments, &nsd->rx_fragments);
1035         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1036                            pf->stat_offsets_loaded,
1037                            &osd->rx_oversize, &nsd->rx_oversize);
1038         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1039                            pf->stat_offsets_loaded,
1040                            &osd->rx_jabber, &nsd->rx_jabber);
1041
1042         /* FDIR stats */
1043         i40e_stat_update32(hw,
1044                            I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(pf->hw.pf_id)),
1045                            pf->stat_offsets_loaded,
1046                            &osd->fd_atr_match, &nsd->fd_atr_match);
1047         i40e_stat_update32(hw,
1048                            I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(pf->hw.pf_id)),
1049                            pf->stat_offsets_loaded,
1050                            &osd->fd_sb_match, &nsd->fd_sb_match);
1051         i40e_stat_update32(hw,
1052                       I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id)),
1053                       pf->stat_offsets_loaded,
1054                       &osd->fd_atr_tunnel_match, &nsd->fd_atr_tunnel_match);
1055
1056         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1057         nsd->tx_lpi_status =
1058                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1059                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1060         nsd->rx_lpi_status =
1061                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1062                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1063         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1064                            pf->stat_offsets_loaded,
1065                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1066         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1067                            pf->stat_offsets_loaded,
1068                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1069
1070         if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1071             !(pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED))
1072                 nsd->fd_sb_status = true;
1073         else
1074                 nsd->fd_sb_status = false;
1075
1076         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1077             !(pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED))
1078                 nsd->fd_atr_status = true;
1079         else
1080                 nsd->fd_atr_status = false;
1081
1082         pf->stat_offsets_loaded = true;
1083 }
1084
1085 /**
1086  * i40e_update_stats - Update the various statistics counters.
1087  * @vsi: the VSI to be updated
1088  *
1089  * Update the various stats for this VSI and its related entities.
1090  **/
1091 void i40e_update_stats(struct i40e_vsi *vsi)
1092 {
1093         struct i40e_pf *pf = vsi->back;
1094
1095         if (vsi == pf->vsi[pf->lan_vsi])
1096                 i40e_update_pf_stats(pf);
1097
1098         i40e_update_vsi_stats(vsi);
1099 }
1100
1101 /**
1102  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1103  * @vsi: the VSI to be searched
1104  * @macaddr: the MAC address
1105  * @vlan: the vlan
1106  *
1107  * Returns ptr to the filter object or NULL
1108  **/
1109 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1110                                                 const u8 *macaddr, s16 vlan)
1111 {
1112         struct i40e_mac_filter *f;
1113         u64 key;
1114
1115         if (!vsi || !macaddr)
1116                 return NULL;
1117
1118         key = i40e_addr_to_hkey(macaddr);
1119         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1120                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1121                     (vlan == f->vlan))
1122                         return f;
1123         }
1124         return NULL;
1125 }
1126
1127 /**
1128  * i40e_find_mac - Find a mac addr in the macvlan filters list
1129  * @vsi: the VSI to be searched
1130  * @macaddr: the MAC address we are searching for
1131  *
1132  * Returns the first filter with the provided MAC address or NULL if
1133  * MAC address was not found
1134  **/
1135 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1136 {
1137         struct i40e_mac_filter *f;
1138         u64 key;
1139
1140         if (!vsi || !macaddr)
1141                 return NULL;
1142
1143         key = i40e_addr_to_hkey(macaddr);
1144         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1145                 if ((ether_addr_equal(macaddr, f->macaddr)))
1146                         return f;
1147         }
1148         return NULL;
1149 }
1150
1151 /**
1152  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1153  * @vsi: the VSI to be searched
1154  *
1155  * Returns true if VSI is in vlan mode or false otherwise
1156  **/
1157 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1158 {
1159         /* If we have a PVID, always operate in VLAN mode */
1160         if (vsi->info.pvid)
1161                 return true;
1162
1163         /* We need to operate in VLAN mode whenever we have any filters with
1164          * a VLAN other than I40E_VLAN_ALL. We could check the table each
1165          * time, incurring search cost repeatedly. However, we can notice two
1166          * things:
1167          *
1168          * 1) the only place where we can gain a VLAN filter is in
1169          *    i40e_add_filter.
1170          *
1171          * 2) the only place where filters are actually removed is in
1172          *    i40e_sync_filters_subtask.
1173          *
1174          * Thus, we can simply use a boolean value, has_vlan_filters which we
1175          * will set to true when we add a VLAN filter in i40e_add_filter. Then
1176          * we have to perform the full search after deleting filters in
1177          * i40e_sync_filters_subtask, but we already have to search
1178          * filters here and can perform the check at the same time. This
1179          * results in avoiding embedding a loop for VLAN mode inside another
1180          * loop over all the filters, and should maintain correctness as noted
1181          * above.
1182          */
1183         return vsi->has_vlan_filter;
1184 }
1185
1186 /**
1187  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1188  * @vsi: the VSI to configure
1189  * @tmp_add_list: list of filters ready to be added
1190  * @tmp_del_list: list of filters ready to be deleted
1191  * @vlan_filters: the number of active VLAN filters
1192  *
1193  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1194  * behave as expected. If we have any active VLAN filters remaining or about
1195  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1196  * so that they only match against untagged traffic. If we no longer have any
1197  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1198  * so that they match against both tagged and untagged traffic. In this way,
1199  * we ensure that we correctly receive the desired traffic. This ensures that
1200  * when we have an active VLAN we will receive only untagged traffic and
1201  * traffic matching active VLANs. If we have no active VLANs then we will
1202  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1203  *
1204  * Finally, in a similar fashion, this function also corrects filters when
1205  * there is an active PVID assigned to this VSI.
1206  *
1207  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1208  *
1209  * This function is only expected to be called from within
1210  * i40e_sync_vsi_filters.
1211  *
1212  * NOTE: This function expects to be called while under the
1213  * mac_filter_hash_lock
1214  */
1215 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1216                                          struct hlist_head *tmp_add_list,
1217                                          struct hlist_head *tmp_del_list,
1218                                          int vlan_filters)
1219 {
1220         s16 pvid = le16_to_cpu(vsi->info.pvid);
1221         struct i40e_mac_filter *f, *add_head;
1222         struct i40e_new_mac_filter *new;
1223         struct hlist_node *h;
1224         int bkt, new_vlan;
1225
1226         /* To determine if a particular filter needs to be replaced we
1227          * have the three following conditions:
1228          *
1229          * a) if we have a PVID assigned, then all filters which are
1230          *    not marked as VLAN=PVID must be replaced with filters that
1231          *    are.
1232          * b) otherwise, if we have any active VLANS, all filters
1233          *    which are marked as VLAN=-1 must be replaced with
1234          *    filters marked as VLAN=0
1235          * c) finally, if we do not have any active VLANS, all filters
1236          *    which are marked as VLAN=0 must be replaced with filters
1237          *    marked as VLAN=-1
1238          */
1239
1240         /* Update the filters about to be added in place */
1241         hlist_for_each_entry(new, tmp_add_list, hlist) {
1242                 if (pvid && new->f->vlan != pvid)
1243                         new->f->vlan = pvid;
1244                 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1245                         new->f->vlan = 0;
1246                 else if (!vlan_filters && new->f->vlan == 0)
1247                         new->f->vlan = I40E_VLAN_ANY;
1248         }
1249
1250         /* Update the remaining active filters */
1251         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1252                 /* Combine the checks for whether a filter needs to be changed
1253                  * and then determine the new VLAN inside the if block, in
1254                  * order to avoid duplicating code for adding the new filter
1255                  * then deleting the old filter.
1256                  */
1257                 if ((pvid && f->vlan != pvid) ||
1258                     (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1259                     (!vlan_filters && f->vlan == 0)) {
1260                         /* Determine the new vlan we will be adding */
1261                         if (pvid)
1262                                 new_vlan = pvid;
1263                         else if (vlan_filters)
1264                                 new_vlan = 0;
1265                         else
1266                                 new_vlan = I40E_VLAN_ANY;
1267
1268                         /* Create the new filter */
1269                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1270                         if (!add_head)
1271                                 return -ENOMEM;
1272
1273                         /* Create a temporary i40e_new_mac_filter */
1274                         new = kzalloc(sizeof(*new), GFP_ATOMIC);
1275                         if (!new)
1276                                 return -ENOMEM;
1277
1278                         new->f = add_head;
1279                         new->state = add_head->state;
1280
1281                         /* Add the new filter to the tmp list */
1282                         hlist_add_head(&new->hlist, tmp_add_list);
1283
1284                         /* Put the original filter into the delete list */
1285                         f->state = I40E_FILTER_REMOVE;
1286                         hash_del(&f->hlist);
1287                         hlist_add_head(&f->hlist, tmp_del_list);
1288                 }
1289         }
1290
1291         vsi->has_vlan_filter = !!vlan_filters;
1292
1293         return 0;
1294 }
1295
1296 /**
1297  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1298  * @vsi: the PF Main VSI - inappropriate for any other VSI
1299  * @macaddr: the MAC address
1300  *
1301  * Remove whatever filter the firmware set up so the driver can manage
1302  * its own filtering intelligently.
1303  **/
1304 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1305 {
1306         struct i40e_aqc_remove_macvlan_element_data element;
1307         struct i40e_pf *pf = vsi->back;
1308
1309         /* Only appropriate for the PF main VSI */
1310         if (vsi->type != I40E_VSI_MAIN)
1311                 return;
1312
1313         memset(&element, 0, sizeof(element));
1314         ether_addr_copy(element.mac_addr, macaddr);
1315         element.vlan_tag = 0;
1316         /* Ignore error returns, some firmware does it this way... */
1317         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1318         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1319
1320         memset(&element, 0, sizeof(element));
1321         ether_addr_copy(element.mac_addr, macaddr);
1322         element.vlan_tag = 0;
1323         /* ...and some firmware does it this way. */
1324         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1325                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1326         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1327 }
1328
1329 /**
1330  * i40e_add_filter - Add a mac/vlan filter to the VSI
1331  * @vsi: the VSI to be searched
1332  * @macaddr: the MAC address
1333  * @vlan: the vlan
1334  *
1335  * Returns ptr to the filter object or NULL when no memory available.
1336  *
1337  * NOTE: This function is expected to be called with mac_filter_hash_lock
1338  * being held.
1339  **/
1340 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1341                                         const u8 *macaddr, s16 vlan)
1342 {
1343         struct i40e_mac_filter *f;
1344         u64 key;
1345
1346         if (!vsi || !macaddr)
1347                 return NULL;
1348
1349         f = i40e_find_filter(vsi, macaddr, vlan);
1350         if (!f) {
1351                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1352                 if (!f)
1353                         return NULL;
1354
1355                 /* Update the boolean indicating if we need to function in
1356                  * VLAN mode.
1357                  */
1358                 if (vlan >= 0)
1359                         vsi->has_vlan_filter = true;
1360
1361                 ether_addr_copy(f->macaddr, macaddr);
1362                 f->vlan = vlan;
1363                 /* If we're in overflow promisc mode, set the state directly
1364                  * to failed, so we don't bother to try sending the filter
1365                  * to the hardware.
1366                  */
1367                 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state))
1368                         f->state = I40E_FILTER_FAILED;
1369                 else
1370                         f->state = I40E_FILTER_NEW;
1371                 INIT_HLIST_NODE(&f->hlist);
1372
1373                 key = i40e_addr_to_hkey(macaddr);
1374                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1375
1376                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1377                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1378         }
1379
1380         /* If we're asked to add a filter that has been marked for removal, it
1381          * is safe to simply restore it to active state. __i40e_del_filter
1382          * will have simply deleted any filters which were previously marked
1383          * NEW or FAILED, so if it is currently marked REMOVE it must have
1384          * previously been ACTIVE. Since we haven't yet run the sync filters
1385          * task, just restore this filter to the ACTIVE state so that the
1386          * sync task leaves it in place
1387          */
1388         if (f->state == I40E_FILTER_REMOVE)
1389                 f->state = I40E_FILTER_ACTIVE;
1390
1391         return f;
1392 }
1393
1394 /**
1395  * __i40e_del_filter - Remove a specific filter from the VSI
1396  * @vsi: VSI to remove from
1397  * @f: the filter to remove from the list
1398  *
1399  * This function should be called instead of i40e_del_filter only if you know
1400  * the exact filter you will remove already, such as via i40e_find_filter or
1401  * i40e_find_mac.
1402  *
1403  * NOTE: This function is expected to be called with mac_filter_hash_lock
1404  * being held.
1405  * ANOTHER NOTE: This function MUST be called from within the context of
1406  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1407  * instead of list_for_each_entry().
1408  **/
1409 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1410 {
1411         if (!f)
1412                 return;
1413
1414         /* If the filter was never added to firmware then we can just delete it
1415          * directly and we don't want to set the status to remove or else an
1416          * admin queue command will unnecessarily fire.
1417          */
1418         if ((f->state == I40E_FILTER_FAILED) ||
1419             (f->state == I40E_FILTER_NEW)) {
1420                 hash_del(&f->hlist);
1421                 kfree(f);
1422         } else {
1423                 f->state = I40E_FILTER_REMOVE;
1424         }
1425
1426         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1427         vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1428 }
1429
1430 /**
1431  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1432  * @vsi: the VSI to be searched
1433  * @macaddr: the MAC address
1434  * @vlan: the VLAN
1435  *
1436  * NOTE: This function is expected to be called with mac_filter_hash_lock
1437  * being held.
1438  * ANOTHER NOTE: This function MUST be called from within the context of
1439  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1440  * instead of list_for_each_entry().
1441  **/
1442 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1443 {
1444         struct i40e_mac_filter *f;
1445
1446         if (!vsi || !macaddr)
1447                 return;
1448
1449         f = i40e_find_filter(vsi, macaddr, vlan);
1450         __i40e_del_filter(vsi, f);
1451 }
1452
1453 /**
1454  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1455  * @vsi: the VSI to be searched
1456  * @macaddr: the mac address to be filtered
1457  *
1458  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1459  * go through all the macvlan filters and add a macvlan filter for each
1460  * unique vlan that already exists. If a PVID has been assigned, instead only
1461  * add the macaddr to that VLAN.
1462  *
1463  * Returns last filter added on success, else NULL
1464  **/
1465 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1466                                             const u8 *macaddr)
1467 {
1468         struct i40e_mac_filter *f, *add = NULL;
1469         struct hlist_node *h;
1470         int bkt;
1471
1472         if (vsi->info.pvid)
1473                 return i40e_add_filter(vsi, macaddr,
1474                                        le16_to_cpu(vsi->info.pvid));
1475
1476         if (!i40e_is_vsi_in_vlan(vsi))
1477                 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1478
1479         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1480                 if (f->state == I40E_FILTER_REMOVE)
1481                         continue;
1482                 add = i40e_add_filter(vsi, macaddr, f->vlan);
1483                 if (!add)
1484                         return NULL;
1485         }
1486
1487         return add;
1488 }
1489
1490 /**
1491  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1492  * @vsi: the VSI to be searched
1493  * @macaddr: the mac address to be removed
1494  *
1495  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1496  * associated with.
1497  *
1498  * Returns 0 for success, or error
1499  **/
1500 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1501 {
1502         struct i40e_mac_filter *f;
1503         struct hlist_node *h;
1504         bool found = false;
1505         int bkt;
1506
1507         WARN(!spin_is_locked(&vsi->mac_filter_hash_lock),
1508              "Missing mac_filter_hash_lock\n");
1509         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1510                 if (ether_addr_equal(macaddr, f->macaddr)) {
1511                         __i40e_del_filter(vsi, f);
1512                         found = true;
1513                 }
1514         }
1515
1516         if (found)
1517                 return 0;
1518         else
1519                 return -ENOENT;
1520 }
1521
1522 /**
1523  * i40e_set_mac - NDO callback to set mac address
1524  * @netdev: network interface device structure
1525  * @p: pointer to an address structure
1526  *
1527  * Returns 0 on success, negative on failure
1528  **/
1529 static int i40e_set_mac(struct net_device *netdev, void *p)
1530 {
1531         struct i40e_netdev_priv *np = netdev_priv(netdev);
1532         struct i40e_vsi *vsi = np->vsi;
1533         struct i40e_pf *pf = vsi->back;
1534         struct i40e_hw *hw = &pf->hw;
1535         struct sockaddr *addr = p;
1536
1537         if (!is_valid_ether_addr(addr->sa_data))
1538                 return -EADDRNOTAVAIL;
1539
1540         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1541                 netdev_info(netdev, "already using mac address %pM\n",
1542                             addr->sa_data);
1543                 return 0;
1544         }
1545
1546         if (test_bit(__I40E_VSI_DOWN, vsi->back->state) ||
1547             test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state))
1548                 return -EADDRNOTAVAIL;
1549
1550         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1551                 netdev_info(netdev, "returning to hw mac address %pM\n",
1552                             hw->mac.addr);
1553         else
1554                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1555
1556         spin_lock_bh(&vsi->mac_filter_hash_lock);
1557         i40e_del_mac_filter(vsi, netdev->dev_addr);
1558         i40e_add_mac_filter(vsi, addr->sa_data);
1559         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1560         ether_addr_copy(netdev->dev_addr, addr->sa_data);
1561         if (vsi->type == I40E_VSI_MAIN) {
1562                 i40e_status ret;
1563
1564                 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1565                                                 I40E_AQC_WRITE_TYPE_LAA_WOL,
1566                                                 addr->sa_data, NULL);
1567                 if (ret)
1568                         netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1569                                     i40e_stat_str(hw, ret),
1570                                     i40e_aq_str(hw, hw->aq.asq_last_status));
1571         }
1572
1573         /* schedule our worker thread which will take care of
1574          * applying the new filter changes
1575          */
1576         i40e_service_event_schedule(vsi->back);
1577         return 0;
1578 }
1579
1580 /**
1581  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1582  * @vsi: the VSI being setup
1583  * @ctxt: VSI context structure
1584  * @enabled_tc: Enabled TCs bitmap
1585  * @is_add: True if called before Add VSI
1586  *
1587  * Setup VSI queue mapping for enabled traffic classes.
1588  **/
1589 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1590                                      struct i40e_vsi_context *ctxt,
1591                                      u8 enabled_tc,
1592                                      bool is_add)
1593 {
1594         struct i40e_pf *pf = vsi->back;
1595         u16 sections = 0;
1596         u8 netdev_tc = 0;
1597         u16 numtc = 0;
1598         u16 qcount;
1599         u8 offset;
1600         u16 qmap;
1601         int i;
1602         u16 num_tc_qps = 0;
1603
1604         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1605         offset = 0;
1606
1607         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1608                 /* Find numtc from enabled TC bitmap */
1609                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1610                         if (enabled_tc & BIT(i)) /* TC is enabled */
1611                                 numtc++;
1612                 }
1613                 if (!numtc) {
1614                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1615                         numtc = 1;
1616                 }
1617         } else {
1618                 /* At least TC0 is enabled in case of non-DCB case */
1619                 numtc = 1;
1620         }
1621
1622         vsi->tc_config.numtc = numtc;
1623         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1624         /* Number of queues per enabled TC */
1625         qcount = vsi->alloc_queue_pairs;
1626
1627         num_tc_qps = qcount / numtc;
1628         num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf));
1629
1630         /* Setup queue offset/count for all TCs for given VSI */
1631         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1632                 /* See if the given TC is enabled for the given VSI */
1633                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1634                         /* TC is enabled */
1635                         int pow, num_qps;
1636
1637                         switch (vsi->type) {
1638                         case I40E_VSI_MAIN:
1639                                 qcount = min_t(int, pf->alloc_rss_size,
1640                                                num_tc_qps);
1641                                 break;
1642                         case I40E_VSI_FDIR:
1643                         case I40E_VSI_SRIOV:
1644                         case I40E_VSI_VMDQ2:
1645                         default:
1646                                 qcount = num_tc_qps;
1647                                 WARN_ON(i != 0);
1648                                 break;
1649                         }
1650                         vsi->tc_config.tc_info[i].qoffset = offset;
1651                         vsi->tc_config.tc_info[i].qcount = qcount;
1652
1653                         /* find the next higher power-of-2 of num queue pairs */
1654                         num_qps = qcount;
1655                         pow = 0;
1656                         while (num_qps && (BIT_ULL(pow) < qcount)) {
1657                                 pow++;
1658                                 num_qps >>= 1;
1659                         }
1660
1661                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1662                         qmap =
1663                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1664                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1665
1666                         offset += qcount;
1667                 } else {
1668                         /* TC is not enabled so set the offset to
1669                          * default queue and allocate one queue
1670                          * for the given TC.
1671                          */
1672                         vsi->tc_config.tc_info[i].qoffset = 0;
1673                         vsi->tc_config.tc_info[i].qcount = 1;
1674                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1675
1676                         qmap = 0;
1677                 }
1678                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1679         }
1680
1681         /* Set actual Tx/Rx queue pairs */
1682         vsi->num_queue_pairs = offset;
1683         if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) {
1684                 if (vsi->req_queue_pairs > 0)
1685                         vsi->num_queue_pairs = vsi->req_queue_pairs;
1686                 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1687                         vsi->num_queue_pairs = pf->num_lan_msix;
1688         }
1689
1690         /* Scheduler section valid can only be set for ADD VSI */
1691         if (is_add) {
1692                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1693
1694                 ctxt->info.up_enable_bits = enabled_tc;
1695         }
1696         if (vsi->type == I40E_VSI_SRIOV) {
1697                 ctxt->info.mapping_flags |=
1698                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1699                 for (i = 0; i < vsi->num_queue_pairs; i++)
1700                         ctxt->info.queue_mapping[i] =
1701                                                cpu_to_le16(vsi->base_queue + i);
1702         } else {
1703                 ctxt->info.mapping_flags |=
1704                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1705                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1706         }
1707         ctxt->info.valid_sections |= cpu_to_le16(sections);
1708 }
1709
1710 /**
1711  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
1712  * @netdev: the netdevice
1713  * @addr: address to add
1714  *
1715  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1716  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1717  */
1718 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
1719 {
1720         struct i40e_netdev_priv *np = netdev_priv(netdev);
1721         struct i40e_vsi *vsi = np->vsi;
1722
1723         if (i40e_add_mac_filter(vsi, addr))
1724                 return 0;
1725         else
1726                 return -ENOMEM;
1727 }
1728
1729 /**
1730  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1731  * @netdev: the netdevice
1732  * @addr: address to add
1733  *
1734  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1735  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1736  */
1737 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
1738 {
1739         struct i40e_netdev_priv *np = netdev_priv(netdev);
1740         struct i40e_vsi *vsi = np->vsi;
1741
1742         i40e_del_mac_filter(vsi, addr);
1743
1744         return 0;
1745 }
1746
1747 /**
1748  * i40e_set_rx_mode - NDO callback to set the netdev filters
1749  * @netdev: network interface device structure
1750  **/
1751 static void i40e_set_rx_mode(struct net_device *netdev)
1752 {
1753         struct i40e_netdev_priv *np = netdev_priv(netdev);
1754         struct i40e_vsi *vsi = np->vsi;
1755
1756         spin_lock_bh(&vsi->mac_filter_hash_lock);
1757
1758         __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1759         __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1760
1761         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1762
1763         /* check for other flag changes */
1764         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1765                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1766                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1767         }
1768
1769         /* schedule our worker thread which will take care of
1770          * applying the new filter changes
1771          */
1772         i40e_service_event_schedule(vsi->back);
1773 }
1774
1775 /**
1776  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
1777  * @vsi: Pointer to VSI struct
1778  * @from: Pointer to list which contains MAC filter entries - changes to
1779  *        those entries needs to be undone.
1780  *
1781  * MAC filter entries from this list were slated for deletion.
1782  **/
1783 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
1784                                          struct hlist_head *from)
1785 {
1786         struct i40e_mac_filter *f;
1787         struct hlist_node *h;
1788
1789         hlist_for_each_entry_safe(f, h, from, hlist) {
1790                 u64 key = i40e_addr_to_hkey(f->macaddr);
1791
1792                 /* Move the element back into MAC filter list*/
1793                 hlist_del(&f->hlist);
1794                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1795         }
1796 }
1797
1798 /**
1799  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
1800  * @vsi: Pointer to vsi struct
1801  * @from: Pointer to list which contains MAC filter entries - changes to
1802  *        those entries needs to be undone.
1803  *
1804  * MAC filter entries from this list were slated for addition.
1805  **/
1806 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
1807                                          struct hlist_head *from)
1808 {
1809         struct i40e_new_mac_filter *new;
1810         struct hlist_node *h;
1811
1812         hlist_for_each_entry_safe(new, h, from, hlist) {
1813                 /* We can simply free the wrapper structure */
1814                 hlist_del(&new->hlist);
1815                 kfree(new);
1816         }
1817 }
1818
1819 /**
1820  * i40e_next_entry - Get the next non-broadcast filter from a list
1821  * @next: pointer to filter in list
1822  *
1823  * Returns the next non-broadcast filter in the list. Required so that we
1824  * ignore broadcast filters within the list, since these are not handled via
1825  * the normal firmware update path.
1826  */
1827 static
1828 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
1829 {
1830         hlist_for_each_entry_continue(next, hlist) {
1831                 if (!is_broadcast_ether_addr(next->f->macaddr))
1832                         return next;
1833         }
1834
1835         return NULL;
1836 }
1837
1838 /**
1839  * i40e_update_filter_state - Update filter state based on return data
1840  * from firmware
1841  * @count: Number of filters added
1842  * @add_list: return data from fw
1843  * @head: pointer to first filter in current batch
1844  *
1845  * MAC filter entries from list were slated to be added to device. Returns
1846  * number of successful filters. Note that 0 does NOT mean success!
1847  **/
1848 static int
1849 i40e_update_filter_state(int count,
1850                          struct i40e_aqc_add_macvlan_element_data *add_list,
1851                          struct i40e_new_mac_filter *add_head)
1852 {
1853         int retval = 0;
1854         int i;
1855
1856         for (i = 0; i < count; i++) {
1857                 /* Always check status of each filter. We don't need to check
1858                  * the firmware return status because we pre-set the filter
1859                  * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
1860                  * request to the adminq. Thus, if it no longer matches then
1861                  * we know the filter is active.
1862                  */
1863                 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
1864                         add_head->state = I40E_FILTER_FAILED;
1865                 } else {
1866                         add_head->state = I40E_FILTER_ACTIVE;
1867                         retval++;
1868                 }
1869
1870                 add_head = i40e_next_filter(add_head);
1871                 if (!add_head)
1872                         break;
1873         }
1874
1875         return retval;
1876 }
1877
1878 /**
1879  * i40e_aqc_del_filters - Request firmware to delete a set of filters
1880  * @vsi: ptr to the VSI
1881  * @vsi_name: name to display in messages
1882  * @list: the list of filters to send to firmware
1883  * @num_del: the number of filters to delete
1884  * @retval: Set to -EIO on failure to delete
1885  *
1886  * Send a request to firmware via AdminQ to delete a set of filters. Uses
1887  * *retval instead of a return value so that success does not force ret_val to
1888  * be set to 0. This ensures that a sequence of calls to this function
1889  * preserve the previous value of *retval on successful delete.
1890  */
1891 static
1892 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
1893                           struct i40e_aqc_remove_macvlan_element_data *list,
1894                           int num_del, int *retval)
1895 {
1896         struct i40e_hw *hw = &vsi->back->hw;
1897         i40e_status aq_ret;
1898         int aq_err;
1899
1900         aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
1901         aq_err = hw->aq.asq_last_status;
1902
1903         /* Explicitly ignore and do not report when firmware returns ENOENT */
1904         if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
1905                 *retval = -EIO;
1906                 dev_info(&vsi->back->pdev->dev,
1907                          "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
1908                          vsi_name, i40e_stat_str(hw, aq_ret),
1909                          i40e_aq_str(hw, aq_err));
1910         }
1911 }
1912
1913 /**
1914  * i40e_aqc_add_filters - Request firmware to add a set of filters
1915  * @vsi: ptr to the VSI
1916  * @vsi_name: name to display in messages
1917  * @list: the list of filters to send to firmware
1918  * @add_head: Position in the add hlist
1919  * @num_add: the number of filters to add
1920  * @promisc_change: set to true on exit if promiscuous mode was forced on
1921  *
1922  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
1923  * promisc_changed to true if the firmware has run out of space for more
1924  * filters.
1925  */
1926 static
1927 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
1928                           struct i40e_aqc_add_macvlan_element_data *list,
1929                           struct i40e_new_mac_filter *add_head,
1930                           int num_add, bool *promisc_changed)
1931 {
1932         struct i40e_hw *hw = &vsi->back->hw;
1933         int aq_err, fcnt;
1934
1935         i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
1936         aq_err = hw->aq.asq_last_status;
1937         fcnt = i40e_update_filter_state(num_add, list, add_head);
1938
1939         if (fcnt != num_add) {
1940                 *promisc_changed = true;
1941                 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
1942                 dev_warn(&vsi->back->pdev->dev,
1943                          "Error %s adding RX filters on %s, promiscuous mode forced on\n",
1944                          i40e_aq_str(hw, aq_err),
1945                          vsi_name);
1946         }
1947 }
1948
1949 /**
1950  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
1951  * @vsi: pointer to the VSI
1952  * @f: filter data
1953  *
1954  * This function sets or clears the promiscuous broadcast flags for VLAN
1955  * filters in order to properly receive broadcast frames. Assumes that only
1956  * broadcast filters are passed.
1957  *
1958  * Returns status indicating success or failure;
1959  **/
1960 static i40e_status
1961 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
1962                           struct i40e_mac_filter *f)
1963 {
1964         bool enable = f->state == I40E_FILTER_NEW;
1965         struct i40e_hw *hw = &vsi->back->hw;
1966         i40e_status aq_ret;
1967
1968         if (f->vlan == I40E_VLAN_ANY) {
1969                 aq_ret = i40e_aq_set_vsi_broadcast(hw,
1970                                                    vsi->seid,
1971                                                    enable,
1972                                                    NULL);
1973         } else {
1974                 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
1975                                                             vsi->seid,
1976                                                             enable,
1977                                                             f->vlan,
1978                                                             NULL);
1979         }
1980
1981         if (aq_ret)
1982                 dev_warn(&vsi->back->pdev->dev,
1983                          "Error %s setting broadcast promiscuous mode on %s\n",
1984                          i40e_aq_str(hw, hw->aq.asq_last_status),
1985                          vsi_name);
1986
1987         return aq_ret;
1988 }
1989
1990 /**
1991  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
1992  * @vsi: ptr to the VSI
1993  *
1994  * Push any outstanding VSI filter changes through the AdminQ.
1995  *
1996  * Returns 0 or error value
1997  **/
1998 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
1999 {
2000         struct hlist_head tmp_add_list, tmp_del_list;
2001         struct i40e_mac_filter *f;
2002         struct i40e_new_mac_filter *new, *add_head = NULL;
2003         struct i40e_hw *hw = &vsi->back->hw;
2004         unsigned int failed_filters = 0;
2005         unsigned int vlan_filters = 0;
2006         bool promisc_changed = false;
2007         char vsi_name[16] = "PF";
2008         int filter_list_len = 0;
2009         i40e_status aq_ret = 0;
2010         u32 changed_flags = 0;
2011         struct hlist_node *h;
2012         struct i40e_pf *pf;
2013         int num_add = 0;
2014         int num_del = 0;
2015         int retval = 0;
2016         u16 cmd_flags;
2017         int list_size;
2018         int bkt;
2019
2020         /* empty array typed pointers, kcalloc later */
2021         struct i40e_aqc_add_macvlan_element_data *add_list;
2022         struct i40e_aqc_remove_macvlan_element_data *del_list;
2023
2024         while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2025                 usleep_range(1000, 2000);
2026         pf = vsi->back;
2027
2028         if (vsi->netdev) {
2029                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2030                 vsi->current_netdev_flags = vsi->netdev->flags;
2031         }
2032
2033         INIT_HLIST_HEAD(&tmp_add_list);
2034         INIT_HLIST_HEAD(&tmp_del_list);
2035
2036         if (vsi->type == I40E_VSI_SRIOV)
2037                 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2038         else if (vsi->type != I40E_VSI_MAIN)
2039                 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2040
2041         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2042                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2043
2044                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2045                 /* Create a list of filters to delete. */
2046                 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2047                         if (f->state == I40E_FILTER_REMOVE) {
2048                                 /* Move the element into temporary del_list */
2049                                 hash_del(&f->hlist);
2050                                 hlist_add_head(&f->hlist, &tmp_del_list);
2051
2052                                 /* Avoid counting removed filters */
2053                                 continue;
2054                         }
2055                         if (f->state == I40E_FILTER_NEW) {
2056                                 /* Create a temporary i40e_new_mac_filter */
2057                                 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2058                                 if (!new)
2059                                         goto err_no_memory_locked;
2060
2061                                 /* Store pointer to the real filter */
2062                                 new->f = f;
2063                                 new->state = f->state;
2064
2065                                 /* Add it to the hash list */
2066                                 hlist_add_head(&new->hlist, &tmp_add_list);
2067                         }
2068
2069                         /* Count the number of active (current and new) VLAN
2070                          * filters we have now. Does not count filters which
2071                          * are marked for deletion.
2072                          */
2073                         if (f->vlan > 0)
2074                                 vlan_filters++;
2075                 }
2076
2077                 retval = i40e_correct_mac_vlan_filters(vsi,
2078                                                        &tmp_add_list,
2079                                                        &tmp_del_list,
2080                                                        vlan_filters);
2081                 if (retval)
2082                         goto err_no_memory_locked;
2083
2084                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2085         }
2086
2087         /* Now process 'del_list' outside the lock */
2088         if (!hlist_empty(&tmp_del_list)) {
2089                 filter_list_len = hw->aq.asq_buf_size /
2090                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2091                 list_size = filter_list_len *
2092                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2093                 del_list = kzalloc(list_size, GFP_ATOMIC);
2094                 if (!del_list)
2095                         goto err_no_memory;
2096
2097                 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2098                         cmd_flags = 0;
2099
2100                         /* handle broadcast filters by updating the broadcast
2101                          * promiscuous flag and release filter list.
2102                          */
2103                         if (is_broadcast_ether_addr(f->macaddr)) {
2104                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2105
2106                                 hlist_del(&f->hlist);
2107                                 kfree(f);
2108                                 continue;
2109                         }
2110
2111                         /* add to delete list */
2112                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2113                         if (f->vlan == I40E_VLAN_ANY) {
2114                                 del_list[num_del].vlan_tag = 0;
2115                                 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2116                         } else {
2117                                 del_list[num_del].vlan_tag =
2118                                         cpu_to_le16((u16)(f->vlan));
2119                         }
2120
2121                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2122                         del_list[num_del].flags = cmd_flags;
2123                         num_del++;
2124
2125                         /* flush a full buffer */
2126                         if (num_del == filter_list_len) {
2127                                 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2128                                                      num_del, &retval);
2129                                 memset(del_list, 0, list_size);
2130                                 num_del = 0;
2131                         }
2132                         /* Release memory for MAC filter entries which were
2133                          * synced up with HW.
2134                          */
2135                         hlist_del(&f->hlist);
2136                         kfree(f);
2137                 }
2138
2139                 if (num_del) {
2140                         i40e_aqc_del_filters(vsi, vsi_name, del_list,
2141                                              num_del, &retval);
2142                 }
2143
2144                 kfree(del_list);
2145                 del_list = NULL;
2146         }
2147
2148         if (!hlist_empty(&tmp_add_list)) {
2149                 /* Do all the adds now. */
2150                 filter_list_len = hw->aq.asq_buf_size /
2151                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2152                 list_size = filter_list_len *
2153                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2154                 add_list = kzalloc(list_size, GFP_ATOMIC);
2155                 if (!add_list)
2156                         goto err_no_memory;
2157
2158                 num_add = 0;
2159                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2160                         if (test_bit(__I40E_VSI_OVERFLOW_PROMISC,
2161                                      vsi->state)) {
2162                                 new->state = I40E_FILTER_FAILED;
2163                                 continue;
2164                         }
2165
2166                         /* handle broadcast filters by updating the broadcast
2167                          * promiscuous flag instead of adding a MAC filter.
2168                          */
2169                         if (is_broadcast_ether_addr(new->f->macaddr)) {
2170                                 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2171                                                               new->f))
2172                                         new->state = I40E_FILTER_FAILED;
2173                                 else
2174                                         new->state = I40E_FILTER_ACTIVE;
2175                                 continue;
2176                         }
2177
2178                         /* add to add array */
2179                         if (num_add == 0)
2180                                 add_head = new;
2181                         cmd_flags = 0;
2182                         ether_addr_copy(add_list[num_add].mac_addr,
2183                                         new->f->macaddr);
2184                         if (new->f->vlan == I40E_VLAN_ANY) {
2185                                 add_list[num_add].vlan_tag = 0;
2186                                 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2187                         } else {
2188                                 add_list[num_add].vlan_tag =
2189                                         cpu_to_le16((u16)(new->f->vlan));
2190                         }
2191                         add_list[num_add].queue_number = 0;
2192                         /* set invalid match method for later detection */
2193                         add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2194                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2195                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2196                         num_add++;
2197
2198                         /* flush a full buffer */
2199                         if (num_add == filter_list_len) {
2200                                 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2201                                                      add_head, num_add,
2202                                                      &promisc_changed);
2203                                 memset(add_list, 0, list_size);
2204                                 num_add = 0;
2205                         }
2206                 }
2207                 if (num_add) {
2208                         i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2209                                              num_add, &promisc_changed);
2210                 }
2211                 /* Now move all of the filters from the temp add list back to
2212                  * the VSI's list.
2213                  */
2214                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2215                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2216                         /* Only update the state if we're still NEW */
2217                         if (new->f->state == I40E_FILTER_NEW)
2218                                 new->f->state = new->state;
2219                         hlist_del(&new->hlist);
2220                         kfree(new);
2221                 }
2222                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2223                 kfree(add_list);
2224                 add_list = NULL;
2225         }
2226
2227         /* Determine the number of active and failed filters. */
2228         spin_lock_bh(&vsi->mac_filter_hash_lock);
2229         vsi->active_filters = 0;
2230         hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2231                 if (f->state == I40E_FILTER_ACTIVE)
2232                         vsi->active_filters++;
2233                 else if (f->state == I40E_FILTER_FAILED)
2234                         failed_filters++;
2235         }
2236         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2237
2238         /* If promiscuous mode has changed, we need to calculate a new
2239          * threshold for when we are safe to exit
2240          */
2241         if (promisc_changed)
2242                 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2243
2244         /* Check if we are able to exit overflow promiscuous mode. We can
2245          * safely exit if we didn't just enter, we no longer have any failed
2246          * filters, and we have reduced filters below the threshold value.
2247          */
2248         if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state) &&
2249             !promisc_changed && !failed_filters &&
2250             (vsi->active_filters < vsi->promisc_threshold)) {
2251                 dev_info(&pf->pdev->dev,
2252                          "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2253                          vsi_name);
2254                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2255                 promisc_changed = true;
2256                 vsi->promisc_threshold = 0;
2257         }
2258
2259         /* if the VF is not trusted do not do promisc */
2260         if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2261                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2262                 goto out;
2263         }
2264
2265         /* check for changes in promiscuous modes */
2266         if (changed_flags & IFF_ALLMULTI) {
2267                 bool cur_multipromisc;
2268
2269                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2270                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2271                                                                vsi->seid,
2272                                                                cur_multipromisc,
2273                                                                NULL);
2274                 if (aq_ret) {
2275                         retval = i40e_aq_rc_to_posix(aq_ret,
2276                                                      hw->aq.asq_last_status);
2277                         dev_info(&pf->pdev->dev,
2278                                  "set multi promisc failed on %s, err %s aq_err %s\n",
2279                                  vsi_name,
2280                                  i40e_stat_str(hw, aq_ret),
2281                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2282                 }
2283         }
2284
2285         if ((changed_flags & IFF_PROMISC) || promisc_changed) {
2286                 bool cur_promisc;
2287
2288                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2289                                test_bit(__I40E_VSI_OVERFLOW_PROMISC,
2290                                         vsi->state));
2291                 if ((vsi->type == I40E_VSI_MAIN) &&
2292                     (pf->lan_veb != I40E_NO_VEB) &&
2293                     !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2294                         /* set defport ON for Main VSI instead of true promisc
2295                          * this way we will get all unicast/multicast and VLAN
2296                          * promisc behavior but will not get VF or VMDq traffic
2297                          * replicated on the Main VSI.
2298                          */
2299                         if (pf->cur_promisc != cur_promisc) {
2300                                 pf->cur_promisc = cur_promisc;
2301                                 if (cur_promisc)
2302                                         aq_ret =
2303                                               i40e_aq_set_default_vsi(hw,
2304                                                                       vsi->seid,
2305                                                                       NULL);
2306                                 else
2307                                         aq_ret =
2308                                             i40e_aq_clear_default_vsi(hw,
2309                                                                       vsi->seid,
2310                                                                       NULL);
2311                                 if (aq_ret) {
2312                                         retval = i40e_aq_rc_to_posix(aq_ret,
2313                                                         hw->aq.asq_last_status);
2314                                         dev_info(&pf->pdev->dev,
2315                                                  "Set default VSI failed on %s, err %s, aq_err %s\n",
2316                                                  vsi_name,
2317                                                  i40e_stat_str(hw, aq_ret),
2318                                                  i40e_aq_str(hw,
2319                                                      hw->aq.asq_last_status));
2320                                 }
2321                         }
2322                 } else {
2323                         aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2324                                                           hw,
2325                                                           vsi->seid,
2326                                                           cur_promisc, NULL,
2327                                                           true);
2328                         if (aq_ret) {
2329                                 retval =
2330                                 i40e_aq_rc_to_posix(aq_ret,
2331                                                     hw->aq.asq_last_status);
2332                                 dev_info(&pf->pdev->dev,
2333                                          "set unicast promisc failed on %s, err %s, aq_err %s\n",
2334                                          vsi_name,
2335                                          i40e_stat_str(hw, aq_ret),
2336                                          i40e_aq_str(hw,
2337                                                      hw->aq.asq_last_status));
2338                         }
2339                         aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2340                                                           hw,
2341                                                           vsi->seid,
2342                                                           cur_promisc, NULL);
2343                         if (aq_ret) {
2344                                 retval =
2345                                 i40e_aq_rc_to_posix(aq_ret,
2346                                                     hw->aq.asq_last_status);
2347                                 dev_info(&pf->pdev->dev,
2348                                          "set multicast promisc failed on %s, err %s, aq_err %s\n",
2349                                          vsi_name,
2350                                          i40e_stat_str(hw, aq_ret),
2351                                          i40e_aq_str(hw,
2352                                                      hw->aq.asq_last_status));
2353                         }
2354                 }
2355                 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
2356                                                    vsi->seid,
2357                                                    cur_promisc, NULL);
2358                 if (aq_ret) {
2359                         retval = i40e_aq_rc_to_posix(aq_ret,
2360                                                      pf->hw.aq.asq_last_status);
2361                         dev_info(&pf->pdev->dev,
2362                                  "set brdcast promisc failed, err %s, aq_err %s\n",
2363                                          i40e_stat_str(hw, aq_ret),
2364                                          i40e_aq_str(hw,
2365                                                      hw->aq.asq_last_status));
2366                 }
2367         }
2368 out:
2369         /* if something went wrong then set the changed flag so we try again */
2370         if (retval)
2371                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2372
2373         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2374         return retval;
2375
2376 err_no_memory:
2377         /* Restore elements on the temporary add and delete lists */
2378         spin_lock_bh(&vsi->mac_filter_hash_lock);
2379 err_no_memory_locked:
2380         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2381         i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2382         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2383
2384         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2385         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2386         return -ENOMEM;
2387 }
2388
2389 /**
2390  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2391  * @pf: board private structure
2392  **/
2393 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2394 {
2395         int v;
2396
2397         if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
2398                 return;
2399         pf->flags &= ~I40E_FLAG_FILTER_SYNC;
2400
2401         for (v = 0; v < pf->num_alloc_vsi; v++) {
2402                 if (pf->vsi[v] &&
2403                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) {
2404                         int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2405
2406                         if (ret) {
2407                                 /* come back and try again later */
2408                                 pf->flags |= I40E_FLAG_FILTER_SYNC;
2409                                 break;
2410                         }
2411                 }
2412         }
2413 }
2414
2415 /**
2416  * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2417  * @vsi: the vsi
2418  **/
2419 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
2420 {
2421         if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2422                 return I40E_RXBUFFER_2048;
2423         else
2424                 return I40E_RXBUFFER_3072;
2425 }
2426
2427 /**
2428  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2429  * @netdev: network interface device structure
2430  * @new_mtu: new value for maximum frame size
2431  *
2432  * Returns 0 on success, negative on failure
2433  **/
2434 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2435 {
2436         struct i40e_netdev_priv *np = netdev_priv(netdev);
2437         struct i40e_vsi *vsi = np->vsi;
2438         struct i40e_pf *pf = vsi->back;
2439
2440         if (i40e_enabled_xdp_vsi(vsi)) {
2441                 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2442
2443                 if (frame_size > i40e_max_xdp_frame_size(vsi))
2444                         return -EINVAL;
2445         }
2446
2447         netdev_info(netdev, "changing MTU from %d to %d\n",
2448                     netdev->mtu, new_mtu);
2449         netdev->mtu = new_mtu;
2450         if (netif_running(netdev))
2451                 i40e_vsi_reinit_locked(vsi);
2452         pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED |
2453                       I40E_FLAG_CLIENT_L2_CHANGE);
2454         return 0;
2455 }
2456
2457 /**
2458  * i40e_ioctl - Access the hwtstamp interface
2459  * @netdev: network interface device structure
2460  * @ifr: interface request data
2461  * @cmd: ioctl command
2462  **/
2463 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2464 {
2465         struct i40e_netdev_priv *np = netdev_priv(netdev);
2466         struct i40e_pf *pf = np->vsi->back;
2467
2468         switch (cmd) {
2469         case SIOCGHWTSTAMP:
2470                 return i40e_ptp_get_ts_config(pf, ifr);
2471         case SIOCSHWTSTAMP:
2472                 return i40e_ptp_set_ts_config(pf, ifr);
2473         default:
2474                 return -EOPNOTSUPP;
2475         }
2476 }
2477
2478 /**
2479  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2480  * @vsi: the vsi being adjusted
2481  **/
2482 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2483 {
2484         struct i40e_vsi_context ctxt;
2485         i40e_status ret;
2486
2487         if ((vsi->info.valid_sections &
2488              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2489             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2490                 return;  /* already enabled */
2491
2492         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2493         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2494                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2495
2496         ctxt.seid = vsi->seid;
2497         ctxt.info = vsi->info;
2498         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2499         if (ret) {
2500                 dev_info(&vsi->back->pdev->dev,
2501                          "update vlan stripping failed, err %s aq_err %s\n",
2502                          i40e_stat_str(&vsi->back->hw, ret),
2503                          i40e_aq_str(&vsi->back->hw,
2504                                      vsi->back->hw.aq.asq_last_status));
2505         }
2506 }
2507
2508 /**
2509  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2510  * @vsi: the vsi being adjusted
2511  **/
2512 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2513 {
2514         struct i40e_vsi_context ctxt;
2515         i40e_status ret;
2516
2517         if ((vsi->info.valid_sections &
2518              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2519             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2520              I40E_AQ_VSI_PVLAN_EMOD_MASK))
2521                 return;  /* already disabled */
2522
2523         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2524         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2525                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2526
2527         ctxt.seid = vsi->seid;
2528         ctxt.info = vsi->info;
2529         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2530         if (ret) {
2531                 dev_info(&vsi->back->pdev->dev,
2532                          "update vlan stripping failed, err %s aq_err %s\n",
2533                          i40e_stat_str(&vsi->back->hw, ret),
2534                          i40e_aq_str(&vsi->back->hw,
2535                                      vsi->back->hw.aq.asq_last_status));
2536         }
2537 }
2538
2539 /**
2540  * i40e_vlan_rx_register - Setup or shutdown vlan offload
2541  * @netdev: network interface to be adjusted
2542  * @features: netdev features to test if VLAN offload is enabled or not
2543  **/
2544 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
2545 {
2546         struct i40e_netdev_priv *np = netdev_priv(netdev);
2547         struct i40e_vsi *vsi = np->vsi;
2548
2549         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2550                 i40e_vlan_stripping_enable(vsi);
2551         else
2552                 i40e_vlan_stripping_disable(vsi);
2553 }
2554
2555 /**
2556  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
2557  * @vsi: the vsi being configured
2558  * @vid: vlan id to be added (0 = untagged only , -1 = any)
2559  *
2560  * This is a helper function for adding a new MAC/VLAN filter with the
2561  * specified VLAN for each existing MAC address already in the hash table.
2562  * This function does *not* perform any accounting to update filters based on
2563  * VLAN mode.
2564  *
2565  * NOTE: this function expects to be called while under the
2566  * mac_filter_hash_lock
2567  **/
2568 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2569 {
2570         struct i40e_mac_filter *f, *add_f;
2571         struct hlist_node *h;
2572         int bkt;
2573
2574         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2575                 if (f->state == I40E_FILTER_REMOVE)
2576                         continue;
2577                 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2578                 if (!add_f) {
2579                         dev_info(&vsi->back->pdev->dev,
2580                                  "Could not add vlan filter %d for %pM\n",
2581                                  vid, f->macaddr);
2582                         return -ENOMEM;
2583                 }
2584         }
2585
2586         return 0;
2587 }
2588
2589 /**
2590  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
2591  * @vsi: the VSI being configured
2592  * @vid: VLAN id to be added
2593  **/
2594 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
2595 {
2596         int err;
2597
2598         if (vsi->info.pvid)
2599                 return -EINVAL;
2600
2601         /* The network stack will attempt to add VID=0, with the intention to
2602          * receive priority tagged packets with a VLAN of 0. Our HW receives
2603          * these packets by default when configured to receive untagged
2604          * packets, so we don't need to add a filter for this case.
2605          * Additionally, HW interprets adding a VID=0 filter as meaning to
2606          * receive *only* tagged traffic and stops receiving untagged traffic.
2607          * Thus, we do not want to actually add a filter for VID=0
2608          */
2609         if (!vid)
2610                 return 0;
2611
2612         /* Locked once because all functions invoked below iterates list*/
2613         spin_lock_bh(&vsi->mac_filter_hash_lock);
2614         err = i40e_add_vlan_all_mac(vsi, vid);
2615         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2616         if (err)
2617                 return err;
2618
2619         /* schedule our worker thread which will take care of
2620          * applying the new filter changes
2621          */
2622         i40e_service_event_schedule(vsi->back);
2623         return 0;
2624 }
2625
2626 /**
2627  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
2628  * @vsi: the vsi being configured
2629  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2630  *
2631  * This function should be used to remove all VLAN filters which match the
2632  * given VID. It does not schedule the service event and does not take the
2633  * mac_filter_hash_lock so it may be combined with other operations under
2634  * a single invocation of the mac_filter_hash_lock.
2635  *
2636  * NOTE: this function expects to be called while under the
2637  * mac_filter_hash_lock
2638  */
2639 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2640 {
2641         struct i40e_mac_filter *f;
2642         struct hlist_node *h;
2643         int bkt;
2644
2645         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2646                 if (f->vlan == vid)
2647                         __i40e_del_filter(vsi, f);
2648         }
2649 }
2650
2651 /**
2652  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
2653  * @vsi: the VSI being configured
2654  * @vid: VLAN id to be removed
2655  **/
2656 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
2657 {
2658         if (!vid || vsi->info.pvid)
2659                 return;
2660
2661         spin_lock_bh(&vsi->mac_filter_hash_lock);
2662         i40e_rm_vlan_all_mac(vsi, vid);
2663         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2664
2665         /* schedule our worker thread which will take care of
2666          * applying the new filter changes
2667          */
2668         i40e_service_event_schedule(vsi->back);
2669 }
2670
2671 /**
2672  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2673  * @netdev: network interface to be adjusted
2674  * @vid: vlan id to be added
2675  *
2676  * net_device_ops implementation for adding vlan ids
2677  **/
2678 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2679                                 __always_unused __be16 proto, u16 vid)
2680 {
2681         struct i40e_netdev_priv *np = netdev_priv(netdev);
2682         struct i40e_vsi *vsi = np->vsi;
2683         int ret = 0;
2684
2685         if (vid >= VLAN_N_VID)
2686                 return -EINVAL;
2687
2688         ret = i40e_vsi_add_vlan(vsi, vid);
2689         if (!ret)
2690                 set_bit(vid, vsi->active_vlans);
2691
2692         return ret;
2693 }
2694
2695 /**
2696  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2697  * @netdev: network interface to be adjusted
2698  * @vid: vlan id to be removed
2699  *
2700  * net_device_ops implementation for removing vlan ids
2701  **/
2702 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2703                                  __always_unused __be16 proto, u16 vid)
2704 {
2705         struct i40e_netdev_priv *np = netdev_priv(netdev);
2706         struct i40e_vsi *vsi = np->vsi;
2707
2708         /* return code is ignored as there is nothing a user
2709          * can do about failure to remove and a log message was
2710          * already printed from the other function
2711          */
2712         i40e_vsi_kill_vlan(vsi, vid);
2713
2714         clear_bit(vid, vsi->active_vlans);
2715
2716         return 0;
2717 }
2718
2719 /**
2720  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2721  * @vsi: the vsi being brought back up
2722  **/
2723 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2724 {
2725         u16 vid;
2726
2727         if (!vsi->netdev)
2728                 return;
2729
2730         i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
2731
2732         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2733                 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
2734                                      vid);
2735 }
2736
2737 /**
2738  * i40e_vsi_add_pvid - Add pvid for the VSI
2739  * @vsi: the vsi being adjusted
2740  * @vid: the vlan id to set as a PVID
2741  **/
2742 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2743 {
2744         struct i40e_vsi_context ctxt;
2745         i40e_status ret;
2746
2747         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2748         vsi->info.pvid = cpu_to_le16(vid);
2749         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2750                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2751                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2752
2753         ctxt.seid = vsi->seid;
2754         ctxt.info = vsi->info;
2755         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2756         if (ret) {
2757                 dev_info(&vsi->back->pdev->dev,
2758                          "add pvid failed, err %s aq_err %s\n",
2759                          i40e_stat_str(&vsi->back->hw, ret),
2760                          i40e_aq_str(&vsi->back->hw,
2761                                      vsi->back->hw.aq.asq_last_status));
2762                 return -ENOENT;
2763         }
2764
2765         return 0;
2766 }
2767
2768 /**
2769  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
2770  * @vsi: the vsi being adjusted
2771  *
2772  * Just use the vlan_rx_register() service to put it back to normal
2773  **/
2774 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2775 {
2776         i40e_vlan_stripping_disable(vsi);
2777
2778         vsi->info.pvid = 0;
2779 }
2780
2781 /**
2782  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
2783  * @vsi: ptr to the VSI
2784  *
2785  * If this function returns with an error, then it's possible one or
2786  * more of the rings is populated (while the rest are not).  It is the
2787  * callers duty to clean those orphaned rings.
2788  *
2789  * Return 0 on success, negative on failure
2790  **/
2791 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2792 {
2793         int i, err = 0;
2794
2795         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2796                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2797
2798         if (!i40e_enabled_xdp_vsi(vsi))
2799                 return err;
2800
2801         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2802                 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
2803
2804         return err;
2805 }
2806
2807 /**
2808  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
2809  * @vsi: ptr to the VSI
2810  *
2811  * Free VSI's transmit software resources
2812  **/
2813 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
2814 {
2815         int i;
2816
2817         if (vsi->tx_rings) {
2818                 for (i = 0; i < vsi->num_queue_pairs; i++)
2819                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
2820                                 i40e_free_tx_resources(vsi->tx_rings[i]);
2821         }
2822
2823         if (vsi->xdp_rings) {
2824                 for (i = 0; i < vsi->num_queue_pairs; i++)
2825                         if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
2826                                 i40e_free_tx_resources(vsi->xdp_rings[i]);
2827         }
2828 }
2829
2830 /**
2831  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
2832  * @vsi: ptr to the VSI
2833  *
2834  * If this function returns with an error, then it's possible one or
2835  * more of the rings is populated (while the rest are not).  It is the
2836  * callers duty to clean those orphaned rings.
2837  *
2838  * Return 0 on success, negative on failure
2839  **/
2840 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
2841 {
2842         int i, err = 0;
2843
2844         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2845                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
2846         return err;
2847 }
2848
2849 /**
2850  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
2851  * @vsi: ptr to the VSI
2852  *
2853  * Free all receive software resources
2854  **/
2855 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
2856 {
2857         int i;
2858
2859         if (!vsi->rx_rings)
2860                 return;
2861
2862         for (i = 0; i < vsi->num_queue_pairs; i++)
2863                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
2864                         i40e_free_rx_resources(vsi->rx_rings[i]);
2865 }
2866
2867 /**
2868  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
2869  * @ring: The Tx ring to configure
2870  *
2871  * This enables/disables XPS for a given Tx descriptor ring
2872  * based on the TCs enabled for the VSI that ring belongs to.
2873  **/
2874 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
2875 {
2876         struct i40e_vsi *vsi = ring->vsi;
2877
2878         if (!ring->q_vector || !ring->netdev)
2879                 return;
2880
2881         if ((vsi->tc_config.numtc <= 1) &&
2882             !test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state)) {
2883                 netif_set_xps_queue(ring->netdev,
2884                                     get_cpu_mask(ring->q_vector->v_idx),
2885                                     ring->queue_index);
2886         }
2887
2888         /* schedule our worker thread which will take care of
2889          * applying the new filter changes
2890          */
2891         i40e_service_event_schedule(vsi->back);
2892 }
2893
2894 /**
2895  * i40e_configure_tx_ring - Configure a transmit ring context and rest
2896  * @ring: The Tx ring to configure
2897  *
2898  * Configure the Tx descriptor ring in the HMC context.
2899  **/
2900 static int i40e_configure_tx_ring(struct i40e_ring *ring)
2901 {
2902         struct i40e_vsi *vsi = ring->vsi;
2903         u16 pf_q = vsi->base_queue + ring->queue_index;
2904         struct i40e_hw *hw = &vsi->back->hw;
2905         struct i40e_hmc_obj_txq tx_ctx;
2906         i40e_status err = 0;
2907         u32 qtx_ctl = 0;
2908
2909         /* some ATR related tx ring init */
2910         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
2911                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
2912                 ring->atr_count = 0;
2913         } else {
2914                 ring->atr_sample_rate = 0;
2915         }
2916
2917         /* configure XPS */
2918         i40e_config_xps_tx_ring(ring);
2919
2920         /* clear the context structure first */
2921         memset(&tx_ctx, 0, sizeof(tx_ctx));
2922
2923         tx_ctx.new_context = 1;
2924         tx_ctx.base = (ring->dma / 128);
2925         tx_ctx.qlen = ring->count;
2926         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
2927                                                I40E_FLAG_FD_ATR_ENABLED));
2928         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
2929         /* FDIR VSI tx ring can still use RS bit and writebacks */
2930         if (vsi->type != I40E_VSI_FDIR)
2931                 tx_ctx.head_wb_ena = 1;
2932         tx_ctx.head_wb_addr = ring->dma +
2933                               (ring->count * sizeof(struct i40e_tx_desc));
2934
2935         /* As part of VSI creation/update, FW allocates certain
2936          * Tx arbitration queue sets for each TC enabled for
2937          * the VSI. The FW returns the handles to these queue
2938          * sets as part of the response buffer to Add VSI,
2939          * Update VSI, etc. AQ commands. It is expected that
2940          * these queue set handles be associated with the Tx
2941          * queues by the driver as part of the TX queue context
2942          * initialization. This has to be done regardless of
2943          * DCB as by default everything is mapped to TC0.
2944          */
2945         tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
2946         tx_ctx.rdylist_act = 0;
2947
2948         /* clear the context in the HMC */
2949         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2950         if (err) {
2951                 dev_info(&vsi->back->pdev->dev,
2952                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
2953                          ring->queue_index, pf_q, err);
2954                 return -ENOMEM;
2955         }
2956
2957         /* set the context in the HMC */
2958         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2959         if (err) {
2960                 dev_info(&vsi->back->pdev->dev,
2961                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
2962                          ring->queue_index, pf_q, err);
2963                 return -ENOMEM;
2964         }
2965
2966         /* Now associate this queue with this PCI function */
2967         if (vsi->type == I40E_VSI_VMDQ2) {
2968                 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
2969                 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
2970                            I40E_QTX_CTL_VFVM_INDX_MASK;
2971         } else {
2972                 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2973         }
2974
2975         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2976                     I40E_QTX_CTL_PF_INDX_MASK);
2977         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
2978         i40e_flush(hw);
2979
2980         /* cache tail off for easier writes later */
2981         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
2982
2983         return 0;
2984 }
2985
2986 /**
2987  * i40e_configure_rx_ring - Configure a receive ring context
2988  * @ring: The Rx ring to configure
2989  *
2990  * Configure the Rx descriptor ring in the HMC context.
2991  **/
2992 static int i40e_configure_rx_ring(struct i40e_ring *ring)
2993 {
2994         struct i40e_vsi *vsi = ring->vsi;
2995         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
2996         u16 pf_q = vsi->base_queue + ring->queue_index;
2997         struct i40e_hw *hw = &vsi->back->hw;
2998         struct i40e_hmc_obj_rxq rx_ctx;
2999         i40e_status err = 0;
3000
3001         ring->state = 0;
3002
3003         /* clear the context structure first */
3004         memset(&rx_ctx, 0, sizeof(rx_ctx));
3005
3006         ring->rx_buf_len = vsi->rx_buf_len;
3007
3008         rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3009                                     BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3010
3011         rx_ctx.base = (ring->dma / 128);
3012         rx_ctx.qlen = ring->count;
3013
3014         /* use 32 byte descriptors */
3015         rx_ctx.dsize = 1;
3016
3017         /* descriptor type is always zero
3018          * rx_ctx.dtype = 0;
3019          */
3020         rx_ctx.hsplit_0 = 0;
3021
3022         rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3023         if (hw->revision_id == 0)
3024                 rx_ctx.lrxqthresh = 0;
3025         else
3026                 rx_ctx.lrxqthresh = 2;
3027         rx_ctx.crcstrip = 1;
3028         rx_ctx.l2tsel = 1;
3029         /* this controls whether VLAN is stripped from inner headers */
3030         rx_ctx.showiv = 0;
3031         /* set the prefena field to 1 because the manual says to */
3032         rx_ctx.prefena = 1;
3033
3034         /* clear the context in the HMC */
3035         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3036         if (err) {
3037                 dev_info(&vsi->back->pdev->dev,
3038                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3039                          ring->queue_index, pf_q, err);
3040                 return -ENOMEM;
3041         }
3042
3043         /* set the context in the HMC */
3044         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3045         if (err) {
3046                 dev_info(&vsi->back->pdev->dev,
3047                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3048                          ring->queue_index, pf_q, err);
3049                 return -ENOMEM;
3050         }
3051
3052         /* configure Rx buffer alignment */
3053         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3054                 clear_ring_build_skb_enabled(ring);
3055         else
3056                 set_ring_build_skb_enabled(ring);
3057
3058         /* cache tail for quicker writes, and clear the reg before use */
3059         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3060         writel(0, ring->tail);
3061
3062         i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3063
3064         return 0;
3065 }
3066
3067 /**
3068  * i40e_vsi_configure_tx - Configure the VSI for Tx
3069  * @vsi: VSI structure describing this set of rings and resources
3070  *
3071  * Configure the Tx VSI for operation.
3072  **/
3073 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3074 {
3075         int err = 0;
3076         u16 i;
3077
3078         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3079                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3080
3081         if (!i40e_enabled_xdp_vsi(vsi))
3082                 return err;
3083
3084         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3085                 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3086
3087         return err;
3088 }
3089
3090 /**
3091  * i40e_vsi_configure_rx - Configure the VSI for Rx
3092  * @vsi: the VSI being configured
3093  *
3094  * Configure the Rx VSI for operation.
3095  **/
3096 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3097 {
3098         int err = 0;
3099         u16 i;
3100
3101         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3102                 vsi->max_frame = I40E_MAX_RXBUFFER;
3103                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
3104 #if (PAGE_SIZE < 8192)
3105         } else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
3106                    (vsi->netdev->mtu <= ETH_DATA_LEN)) {
3107                 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3108                 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3109 #endif
3110         } else {
3111                 vsi->max_frame = I40E_MAX_RXBUFFER;
3112                 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
3113                                                        I40E_RXBUFFER_2048;
3114         }
3115
3116         /* set up individual rings */
3117         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3118                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3119
3120         return err;
3121 }
3122
3123 /**
3124  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3125  * @vsi: ptr to the VSI
3126  **/
3127 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3128 {
3129         struct i40e_ring *tx_ring, *rx_ring;
3130         u16 qoffset, qcount;
3131         int i, n;
3132
3133         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3134                 /* Reset the TC information */
3135                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3136                         rx_ring = vsi->rx_rings[i];
3137                         tx_ring = vsi->tx_rings[i];
3138                         rx_ring->dcb_tc = 0;
3139                         tx_ring->dcb_tc = 0;
3140                 }
3141         }
3142
3143         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3144                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3145                         continue;
3146
3147                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3148                 qcount = vsi->tc_config.tc_info[n].qcount;
3149                 for (i = qoffset; i < (qoffset + qcount); i++) {
3150                         rx_ring = vsi->rx_rings[i];
3151                         tx_ring = vsi->tx_rings[i];
3152                         rx_ring->dcb_tc = n;
3153                         tx_ring->dcb_tc = n;
3154                 }
3155         }
3156 }
3157
3158 /**
3159  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3160  * @vsi: ptr to the VSI
3161  **/
3162 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3163 {
3164         if (vsi->netdev)
3165                 i40e_set_rx_mode(vsi->netdev);
3166 }
3167
3168 /**
3169  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3170  * @vsi: Pointer to the targeted VSI
3171  *
3172  * This function replays the hlist on the hw where all the SB Flow Director
3173  * filters were saved.
3174  **/
3175 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3176 {
3177         struct i40e_fdir_filter *filter;
3178         struct i40e_pf *pf = vsi->back;
3179         struct hlist_node *node;
3180
3181         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3182                 return;
3183
3184         /* Reset FDir counters as we're replaying all existing filters */
3185         pf->fd_tcp4_filter_cnt = 0;
3186         pf->fd_udp4_filter_cnt = 0;
3187         pf->fd_sctp4_filter_cnt = 0;
3188         pf->fd_ip4_filter_cnt = 0;
3189
3190         hlist_for_each_entry_safe(filter, node,
3191                                   &pf->fdir_filter_list, fdir_node) {
3192                 i40e_add_del_fdir(vsi, filter, true);
3193         }
3194 }
3195
3196 /**
3197  * i40e_vsi_configure - Set up the VSI for action
3198  * @vsi: the VSI being configured
3199  **/
3200 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3201 {
3202         int err;
3203
3204         i40e_set_vsi_rx_mode(vsi);
3205         i40e_restore_vlan(vsi);
3206         i40e_vsi_config_dcb_rings(vsi);
3207         err = i40e_vsi_configure_tx(vsi);
3208         if (!err)
3209                 err = i40e_vsi_configure_rx(vsi);
3210
3211         return err;
3212 }
3213
3214 /**
3215  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3216  * @vsi: the VSI being configured
3217  **/
3218 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3219 {
3220         bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3221         struct i40e_pf *pf = vsi->back;
3222         struct i40e_hw *hw = &pf->hw;
3223         u16 vector;
3224         int i, q;
3225         u32 qp;
3226
3227         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3228          * and PFINT_LNKLSTn registers, e.g.:
3229          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3230          */
3231         qp = vsi->base_queue;
3232         vector = vsi->base_vector;
3233         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3234                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3235
3236                 q_vector->itr_countdown = ITR_COUNTDOWN_START;
3237                 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting);
3238                 q_vector->rx.latency_range = I40E_LOW_LATENCY;
3239                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3240                      q_vector->rx.itr);
3241                 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting);
3242                 q_vector->tx.latency_range = I40E_LOW_LATENCY;
3243                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3244                      q_vector->tx.itr);
3245                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3246                      i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3247
3248                 /* Linked list for the queuepairs assigned to this vector */
3249                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3250                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3251                         u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3252                         u32 val;
3253
3254                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3255                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3256                               (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3257                               (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3258                               (I40E_QUEUE_TYPE_TX <<
3259                                I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3260
3261                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3262
3263                         if (has_xdp) {
3264                                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3265                                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3266                                       (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3267                                       (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3268                                       (I40E_QUEUE_TYPE_TX <<
3269                                        I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3270
3271                                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3272                         }
3273
3274                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3275                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3276                               (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3277                               ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3278                               (I40E_QUEUE_TYPE_RX <<
3279                                I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3280
3281                         /* Terminate the linked list */
3282                         if (q == (q_vector->num_ringpairs - 1))
3283                                 val |= (I40E_QUEUE_END_OF_LIST <<
3284                                         I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3285
3286                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3287                         qp++;
3288                 }
3289         }
3290
3291         i40e_flush(hw);
3292 }
3293
3294 /**
3295  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3296  * @hw: ptr to the hardware info
3297  **/
3298 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3299 {
3300         struct i40e_hw *hw = &pf->hw;
3301         u32 val;
3302
3303         /* clear things first */
3304         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3305         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3306
3307         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3308               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3309               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3310               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3311               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3312               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3313               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3314               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3315
3316         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3317                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3318
3319         if (pf->flags & I40E_FLAG_PTP)
3320                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3321
3322         wr32(hw, I40E_PFINT_ICR0_ENA, val);
3323
3324         /* SW_ITR_IDX = 0, but don't change INTENA */
3325         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3326                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3327
3328         /* OTHER_ITR_IDX = 0 */
3329         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3330 }
3331
3332 /**
3333  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3334  * @vsi: the VSI being configured
3335  **/
3336 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3337 {
3338         u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3339         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3340         struct i40e_pf *pf = vsi->back;
3341         struct i40e_hw *hw = &pf->hw;
3342         u32 val;
3343
3344         /* set the ITR configuration */
3345         q_vector->itr_countdown = ITR_COUNTDOWN_START;
3346         q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting);
3347         q_vector->rx.latency_range = I40E_LOW_LATENCY;
3348         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
3349         q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting);
3350         q_vector->tx.latency_range = I40E_LOW_LATENCY;
3351         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
3352
3353         i40e_enable_misc_int_causes(pf);
3354
3355         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3356         wr32(hw, I40E_PFINT_LNKLST0, 0);
3357
3358         /* Associate the queue pair to the vector and enable the queue int */
3359         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                   |
3360               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
3361               (nextqp      << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3362               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3363
3364         wr32(hw, I40E_QINT_RQCTL(0), val);
3365
3366         if (i40e_enabled_xdp_vsi(vsi)) {
3367                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                 |
3368                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
3369                       (I40E_QUEUE_TYPE_TX
3370                        << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3371
3372                wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3373         }
3374
3375         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
3376               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3377               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3378
3379         wr32(hw, I40E_QINT_TQCTL(0), val);
3380         i40e_flush(hw);
3381 }
3382
3383 /**
3384  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3385  * @pf: board private structure
3386  **/
3387 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3388 {
3389         struct i40e_hw *hw = &pf->hw;
3390
3391         wr32(hw, I40E_PFINT_DYN_CTL0,
3392              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3393         i40e_flush(hw);
3394 }
3395
3396 /**
3397  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3398  * @pf: board private structure
3399  * @clearpba: true when all pending interrupt events should be cleared
3400  **/
3401 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf, bool clearpba)
3402 {
3403         struct i40e_hw *hw = &pf->hw;
3404         u32 val;
3405
3406         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
3407               (clearpba ? I40E_PFINT_DYN_CTL0_CLEARPBA_MASK : 0) |
3408               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3409
3410         wr32(hw, I40E_PFINT_DYN_CTL0, val);
3411         i40e_flush(hw);
3412 }
3413
3414 /**
3415  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3416  * @irq: interrupt number
3417  * @data: pointer to a q_vector
3418  **/
3419 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3420 {
3421         struct i40e_q_vector *q_vector = data;
3422
3423         if (!q_vector->tx.ring && !q_vector->rx.ring)
3424                 return IRQ_HANDLED;
3425
3426         napi_schedule_irqoff(&q_vector->napi);
3427
3428         return IRQ_HANDLED;
3429 }
3430
3431 /**
3432  * i40e_irq_affinity_notify - Callback for affinity changes
3433  * @notify: context as to what irq was changed
3434  * @mask: the new affinity mask
3435  *
3436  * This is a callback function used by the irq_set_affinity_notifier function
3437  * so that we may register to receive changes to the irq affinity masks.
3438  **/
3439 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3440                                      const cpumask_t *mask)
3441 {
3442         struct i40e_q_vector *q_vector =
3443                 container_of(notify, struct i40e_q_vector, affinity_notify);
3444
3445         cpumask_copy(&q_vector->affinity_mask, mask);
3446 }
3447
3448 /**
3449  * i40e_irq_affinity_release - Callback for affinity notifier release
3450  * @ref: internal core kernel usage
3451  *
3452  * This is a callback function used by the irq_set_affinity_notifier function
3453  * to inform the current notification subscriber that they will no longer
3454  * receive notifications.
3455  **/
3456 static void i40e_irq_affinity_release(struct kref *ref) {}
3457
3458 /**
3459  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3460  * @vsi: the VSI being configured
3461  * @basename: name for the vector
3462  *
3463  * Allocates MSI-X vectors and requests interrupts from the kernel.
3464  **/
3465 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3466 {
3467         int q_vectors = vsi->num_q_vectors;
3468         struct i40e_pf *pf = vsi->back;
3469         int base = vsi->base_vector;
3470         int rx_int_idx = 0;
3471         int tx_int_idx = 0;
3472         int vector, err;
3473         int irq_num;
3474
3475         for (vector = 0; vector < q_vectors; vector++) {
3476                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3477
3478                 irq_num = pf->msix_entries[base + vector].vector;
3479
3480                 if (q_vector->tx.ring && q_vector->rx.ring) {
3481                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3482                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3483                         tx_int_idx++;
3484                 } else if (q_vector->rx.ring) {
3485                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3486                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
3487                 } else if (q_vector->tx.ring) {
3488                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3489                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
3490                 } else {
3491                         /* skip this unused q_vector */
3492                         continue;
3493                 }
3494                 err = request_irq(irq_num,
3495                                   vsi->irq_handler,
3496                                   0,
3497                                   q_vector->name,
3498                                   q_vector);
3499                 if (err) {
3500                         dev_info(&pf->pdev->dev,
3501                                  "MSIX request_irq failed, error: %d\n", err);
3502                         goto free_queue_irqs;
3503                 }
3504
3505                 /* register for affinity change notifications */
3506                 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3507                 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3508                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3509                 /* get_cpu_mask returns a static constant mask with
3510                  * a permanent lifetime so it's ok to use here.
3511                  */
3512                 irq_set_affinity_hint(irq_num, get_cpu_mask(q_vector->v_idx));
3513         }
3514
3515         vsi->irqs_ready = true;
3516         return 0;
3517
3518 free_queue_irqs:
3519         while (vector) {
3520                 vector--;
3521                 irq_num = pf->msix_entries[base + vector].vector;
3522                 irq_set_affinity_notifier(irq_num, NULL);
3523                 irq_set_affinity_hint(irq_num, NULL);
3524                 free_irq(irq_num, &vsi->q_vectors[vector]);
3525         }
3526         return err;
3527 }
3528
3529 /**
3530  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3531  * @vsi: the VSI being un-configured
3532  **/
3533 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3534 {
3535         struct i40e_pf *pf = vsi->back;
3536         struct i40e_hw *hw = &pf->hw;
3537         int base = vsi->base_vector;
3538         int i;
3539
3540         /* disable interrupt causation from each queue */
3541         for (i = 0; i < vsi->num_queue_pairs; i++) {
3542                 u32 val;
3543
3544                 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
3545                 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
3546                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
3547
3548                 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
3549                 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
3550                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
3551
3552                 if (!i40e_enabled_xdp_vsi(vsi))
3553                         continue;
3554                 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
3555         }
3556
3557         /* disable each interrupt */
3558         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3559                 for (i = vsi->base_vector;
3560                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3561                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3562
3563                 i40e_flush(hw);
3564                 for (i = 0; i < vsi->num_q_vectors; i++)
3565                         synchronize_irq(pf->msix_entries[i + base].vector);
3566         } else {
3567                 /* Legacy and MSI mode - this stops all interrupt handling */
3568                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3569                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3570                 i40e_flush(hw);
3571                 synchronize_irq(pf->pdev->irq);
3572         }
3573 }
3574
3575 /**
3576  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3577  * @vsi: the VSI being configured
3578  **/
3579 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3580 {
3581         struct i40e_pf *pf = vsi->back;
3582         int i;
3583
3584         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3585                 for (i = 0; i < vsi->num_q_vectors; i++)
3586                         i40e_irq_dynamic_enable(vsi, i);
3587         } else {
3588                 i40e_irq_dynamic_enable_icr0(pf, true);
3589         }
3590
3591         i40e_flush(&pf->hw);
3592         return 0;
3593 }
3594
3595 /**
3596  * i40e_stop_misc_vector - Stop the vector that handles non-queue events
3597  * @pf: board private structure
3598  **/
3599 static void i40e_stop_misc_vector(struct i40e_pf *pf)
3600 {
3601         /* Disable ICR 0 */
3602         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3603         i40e_flush(&pf->hw);
3604 }
3605
3606 /**
3607  * i40e_intr - MSI/Legacy and non-queue interrupt handler
3608  * @irq: interrupt number
3609  * @data: pointer to a q_vector
3610  *
3611  * This is the handler used for all MSI/Legacy interrupts, and deals
3612  * with both queue and non-queue interrupts.  This is also used in
3613  * MSIX mode to handle the non-queue interrupts.
3614  **/
3615 static irqreturn_t i40e_intr(int irq, void *data)
3616 {
3617         struct i40e_pf *pf = (struct i40e_pf *)data;
3618         struct i40e_hw *hw = &pf->hw;
3619         irqreturn_t ret = IRQ_NONE;
3620         u32 icr0, icr0_remaining;
3621         u32 val, ena_mask;
3622
3623         icr0 = rd32(hw, I40E_PFINT_ICR0);
3624         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
3625
3626         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
3627         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
3628                 goto enable_intr;
3629
3630         /* if interrupt but no bits showing, must be SWINT */
3631         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
3632             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
3633                 pf->sw_int_count++;
3634
3635         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
3636             (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
3637                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3638                 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
3639                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
3640         }
3641
3642         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
3643         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
3644                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
3645                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3646
3647                 /* We do not have a way to disarm Queue causes while leaving
3648                  * interrupt enabled for all other causes, ideally
3649                  * interrupt should be disabled while we are in NAPI but
3650                  * this is not a performance path and napi_schedule()
3651                  * can deal with rescheduling.
3652                  */
3653                 if (!test_bit(__I40E_DOWN, pf->state))
3654                         napi_schedule_irqoff(&q_vector->napi);
3655         }
3656
3657         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3658                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3659                 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
3660                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
3661         }
3662
3663         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
3664                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3665                 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
3666         }
3667
3668         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
3669                 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
3670                 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
3671         }
3672
3673         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
3674                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
3675                         set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
3676                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
3677                 val = rd32(hw, I40E_GLGEN_RSTAT);
3678                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
3679                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
3680                 if (val == I40E_RESET_CORER) {
3681                         pf->corer_count++;
3682                 } else if (val == I40E_RESET_GLOBR) {
3683                         pf->globr_count++;
3684                 } else if (val == I40E_RESET_EMPR) {
3685                         pf->empr_count++;
3686                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
3687                 }
3688         }
3689
3690         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
3691                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
3692                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
3693                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
3694                          rd32(hw, I40E_PFHMC_ERRORINFO),
3695                          rd32(hw, I40E_PFHMC_ERRORDATA));
3696         }
3697
3698         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
3699                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
3700
3701                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
3702                         icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3703                         i40e_ptp_tx_hwtstamp(pf);
3704                 }
3705         }
3706
3707         /* If a critical error is pending we have no choice but to reset the
3708          * device.
3709          * Report and mask out any remaining unexpected interrupts.
3710          */
3711         icr0_remaining = icr0 & ena_mask;
3712         if (icr0_remaining) {
3713                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
3714                          icr0_remaining);
3715                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
3716                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
3717                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
3718                         dev_info(&pf->pdev->dev, "device will be reset\n");
3719                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
3720                         i40e_service_event_schedule(pf);
3721                 }
3722                 ena_mask &= ~icr0_remaining;
3723         }
3724         ret = IRQ_HANDLED;
3725
3726 enable_intr:
3727         /* re-enable interrupt causes */
3728         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
3729         if (!test_bit(__I40E_DOWN, pf->state)) {
3730                 i40e_service_event_schedule(pf);
3731                 i40e_irq_dynamic_enable_icr0(pf, false);
3732         }
3733
3734         return ret;
3735 }
3736
3737 /**
3738  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
3739  * @tx_ring:  tx ring to clean
3740  * @budget:   how many cleans we're allowed
3741  *
3742  * Returns true if there's any budget left (e.g. the clean is finished)
3743  **/
3744 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
3745 {
3746         struct i40e_vsi *vsi = tx_ring->vsi;
3747         u16 i = tx_ring->next_to_clean;
3748         struct i40e_tx_buffer *tx_buf;
3749         struct i40e_tx_desc *tx_desc;
3750
3751         tx_buf = &tx_ring->tx_bi[i];
3752         tx_desc = I40E_TX_DESC(tx_ring, i);
3753         i -= tx_ring->count;
3754
3755         do {
3756                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
3757
3758                 /* if next_to_watch is not set then there is no work pending */
3759                 if (!eop_desc)
3760                         break;
3761
3762                 /* prevent any other reads prior to eop_desc */
3763                 read_barrier_depends();
3764
3765                 /* if the descriptor isn't done, no work yet to do */
3766                 if (!(eop_desc->cmd_type_offset_bsz &
3767                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
3768                         break;
3769
3770                 /* clear next_to_watch to prevent false hangs */
3771                 tx_buf->next_to_watch = NULL;
3772
3773                 tx_desc->buffer_addr = 0;
3774                 tx_desc->cmd_type_offset_bsz = 0;
3775                 /* move past filter desc */
3776                 tx_buf++;
3777                 tx_desc++;
3778                 i++;
3779                 if (unlikely(!i)) {
3780                         i -= tx_ring->count;
3781                         tx_buf = tx_ring->tx_bi;
3782                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3783                 }
3784                 /* unmap skb header data */
3785                 dma_unmap_single(tx_ring->dev,
3786                                  dma_unmap_addr(tx_buf, dma),
3787                                  dma_unmap_len(tx_buf, len),
3788                                  DMA_TO_DEVICE);
3789                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
3790                         kfree(tx_buf->raw_buf);
3791
3792                 tx_buf->raw_buf = NULL;
3793                 tx_buf->tx_flags = 0;
3794                 tx_buf->next_to_watch = NULL;
3795                 dma_unmap_len_set(tx_buf, len, 0);
3796                 tx_desc->buffer_addr = 0;
3797                 tx_desc->cmd_type_offset_bsz = 0;
3798
3799                 /* move us past the eop_desc for start of next FD desc */
3800                 tx_buf++;
3801                 tx_desc++;
3802                 i++;
3803                 if (unlikely(!i)) {
3804                         i -= tx_ring->count;
3805                         tx_buf = tx_ring->tx_bi;
3806                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3807                 }
3808
3809                 /* update budget accounting */
3810                 budget--;
3811         } while (likely(budget));
3812
3813         i += tx_ring->count;
3814         tx_ring->next_to_clean = i;
3815
3816         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
3817                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
3818
3819         return budget > 0;
3820 }
3821
3822 /**
3823  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
3824  * @irq: interrupt number
3825  * @data: pointer to a q_vector
3826  **/
3827 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
3828 {
3829         struct i40e_q_vector *q_vector = data;
3830         struct i40e_vsi *vsi;
3831
3832         if (!q_vector->tx.ring)
3833                 return IRQ_HANDLED;
3834
3835         vsi = q_vector->tx.ring->vsi;
3836         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
3837
3838         return IRQ_HANDLED;
3839 }
3840
3841 /**
3842  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
3843  * @vsi: the VSI being configured
3844  * @v_idx: vector index
3845  * @qp_idx: queue pair index
3846  **/
3847 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
3848 {
3849         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3850         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
3851         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
3852
3853         tx_ring->q_vector = q_vector;
3854         tx_ring->next = q_vector->tx.ring;
3855         q_vector->tx.ring = tx_ring;
3856         q_vector->tx.count++;
3857
3858         /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
3859         if (i40e_enabled_xdp_vsi(vsi)) {
3860                 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
3861
3862                 xdp_ring->q_vector = q_vector;
3863                 xdp_ring->next = q_vector->tx.ring;
3864                 q_vector->tx.ring = xdp_ring;
3865                 q_vector->tx.count++;
3866         }
3867
3868         rx_ring->q_vector = q_vector;
3869         rx_ring->next = q_vector->rx.ring;
3870         q_vector->rx.ring = rx_ring;
3871         q_vector->rx.count++;
3872 }
3873
3874 /**
3875  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
3876  * @vsi: the VSI being configured
3877  *
3878  * This function maps descriptor rings to the queue-specific vectors
3879  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
3880  * one vector per queue pair, but on a constrained vector budget, we
3881  * group the queue pairs as "efficiently" as possible.
3882  **/
3883 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
3884 {
3885         int qp_remaining = vsi->num_queue_pairs;
3886         int q_vectors = vsi->num_q_vectors;
3887         int num_ringpairs;
3888         int v_start = 0;
3889         int qp_idx = 0;
3890
3891         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
3892          * group them so there are multiple queues per vector.
3893          * It is also important to go through all the vectors available to be
3894          * sure that if we don't use all the vectors, that the remaining vectors
3895          * are cleared. This is especially important when decreasing the
3896          * number of queues in use.
3897          */
3898         for (; v_start < q_vectors; v_start++) {
3899                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
3900
3901                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
3902
3903                 q_vector->num_ringpairs = num_ringpairs;
3904
3905                 q_vector->rx.count = 0;
3906                 q_vector->tx.count = 0;
3907                 q_vector->rx.ring = NULL;
3908                 q_vector->tx.ring = NULL;
3909
3910                 while (num_ringpairs--) {
3911                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
3912                         qp_idx++;
3913                         qp_remaining--;
3914                 }
3915         }
3916 }
3917
3918 /**
3919  * i40e_vsi_request_irq - Request IRQ from the OS
3920  * @vsi: the VSI being configured
3921  * @basename: name for the vector
3922  **/
3923 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
3924 {
3925         struct i40e_pf *pf = vsi->back;
3926         int err;
3927
3928         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
3929                 err = i40e_vsi_request_irq_msix(vsi, basename);
3930         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
3931                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
3932                                   pf->int_name, pf);
3933         else
3934                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
3935                                   pf->int_name, pf);
3936
3937         if (err)
3938                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
3939
3940         return err;
3941 }
3942
3943 #ifdef CONFIG_NET_POLL_CONTROLLER
3944 /**
3945  * i40e_netpoll - A Polling 'interrupt' handler
3946  * @netdev: network interface device structure
3947  *
3948  * This is used by netconsole to send skbs without having to re-enable
3949  * interrupts.  It's not called while the normal interrupt routine is executing.
3950  **/
3951 static void i40e_netpoll(struct net_device *netdev)
3952 {
3953         struct i40e_netdev_priv *np = netdev_priv(netdev);
3954         struct i40e_vsi *vsi = np->vsi;
3955         struct i40e_pf *pf = vsi->back;
3956         int i;
3957
3958         /* if interface is down do nothing */
3959         if (test_bit(__I40E_VSI_DOWN, vsi->state))
3960                 return;
3961
3962         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3963                 for (i = 0; i < vsi->num_q_vectors; i++)
3964                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
3965         } else {
3966                 i40e_intr(pf->pdev->irq, netdev);
3967         }
3968 }
3969 #endif
3970
3971 #define I40E_QTX_ENA_WAIT_COUNT 50
3972
3973 /**
3974  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
3975  * @pf: the PF being configured
3976  * @pf_q: the PF queue
3977  * @enable: enable or disable state of the queue
3978  *
3979  * This routine will wait for the given Tx queue of the PF to reach the
3980  * enabled or disabled state.
3981  * Returns -ETIMEDOUT in case of failing to reach the requested state after
3982  * multiple retries; else will return 0 in case of success.
3983  **/
3984 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
3985 {
3986         int i;
3987         u32 tx_reg;
3988
3989         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
3990                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
3991                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3992                         break;
3993
3994                 usleep_range(10, 20);
3995         }
3996         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
3997                 return -ETIMEDOUT;
3998
3999         return 0;
4000 }
4001
4002 /**
4003  * i40e_control_tx_q - Start or stop a particular Tx queue
4004  * @pf: the PF structure
4005  * @pf_q: the PF queue to configure
4006  * @enable: start or stop the queue
4007  *
4008  * This function enables or disables a single queue. Note that any delay
4009  * required after the operation is expected to be handled by the caller of
4010  * this function.
4011  **/
4012 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4013 {
4014         struct i40e_hw *hw = &pf->hw;
4015         u32 tx_reg;
4016         int i;
4017
4018         /* warn the TX unit of coming changes */
4019         i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4020         if (!enable)
4021                 usleep_range(10, 20);
4022
4023         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4024                 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4025                 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4026                     ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4027                         break;
4028                 usleep_range(1000, 2000);
4029         }
4030
4031         /* Skip if the queue is already in the requested state */
4032         if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4033                 return;
4034
4035         /* turn on/off the queue */
4036         if (enable) {
4037                 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4038                 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4039         } else {
4040                 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4041         }
4042
4043         wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4044 }
4045
4046 /**
4047  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4048  * @seid: VSI SEID
4049  * @pf: the PF structure
4050  * @pf_q: the PF queue to configure
4051  * @is_xdp: true if the queue is used for XDP
4052  * @enable: start or stop the queue
4053  **/
4054 static int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4055                                   bool is_xdp, bool enable)
4056 {
4057         int ret;
4058
4059         i40e_control_tx_q(pf, pf_q, enable);
4060
4061         /* wait for the change to finish */
4062         ret = i40e_pf_txq_wait(pf, pf_q, enable);
4063         if (ret) {
4064                 dev_info(&pf->pdev->dev,
4065                          "VSI seid %d %sTx ring %d %sable timeout\n",
4066                          seid, (is_xdp ? "XDP " : ""), pf_q,
4067                          (enable ? "en" : "dis"));
4068         }
4069
4070         return ret;
4071 }
4072
4073 /**
4074  * i40e_vsi_control_tx - Start or stop a VSI's rings
4075  * @vsi: the VSI being configured
4076  * @enable: start or stop the rings
4077  **/
4078 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
4079 {
4080         struct i40e_pf *pf = vsi->back;
4081         int i, pf_q, ret = 0;
4082
4083         pf_q = vsi->base_queue;
4084         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4085                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4086                                              pf_q,
4087                                              false /*is xdp*/, enable);
4088                 if (ret)
4089                         break;
4090
4091                 if (!i40e_enabled_xdp_vsi(vsi))
4092                         continue;
4093
4094                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4095                                              pf_q + vsi->alloc_queue_pairs,
4096                                              true /*is xdp*/, enable);
4097                 if (ret)
4098                         break;
4099         }
4100
4101         return ret;
4102 }
4103
4104 /**
4105  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4106  * @pf: the PF being configured
4107  * @pf_q: the PF queue
4108  * @enable: enable or disable state of the queue
4109  *
4110  * This routine will wait for the given Rx queue of the PF to reach the
4111  * enabled or disabled state.
4112  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4113  * multiple retries; else will return 0 in case of success.
4114  **/
4115 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4116 {
4117         int i;
4118         u32 rx_reg;
4119
4120         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4121                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4122                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4123                         break;
4124
4125                 usleep_range(10, 20);
4126         }
4127         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4128                 return -ETIMEDOUT;
4129
4130         return 0;
4131 }
4132
4133 /**
4134  * i40e_control_rx_q - Start or stop a particular Rx queue
4135  * @pf: the PF structure
4136  * @pf_q: the PF queue to configure
4137  * @enable: start or stop the queue
4138  *
4139  * This function enables or disables a single queue. Note that any delay
4140  * required after the operation is expected to be handled by the caller of
4141  * this function.
4142  **/
4143 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4144 {
4145         struct i40e_hw *hw = &pf->hw;
4146         u32 rx_reg;
4147         int i;
4148
4149         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4150                 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4151                 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4152                     ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4153                         break;
4154                 usleep_range(1000, 2000);
4155         }
4156
4157         /* Skip if the queue is already in the requested state */
4158         if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4159                 return;
4160
4161         /* turn on/off the queue */
4162         if (enable)
4163                 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4164         else
4165                 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4166
4167         wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4168 }
4169
4170 /**
4171  * i40e_vsi_control_rx - Start or stop a VSI's rings
4172  * @vsi: the VSI being configured
4173  * @enable: start or stop the rings
4174  **/
4175 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
4176 {
4177         struct i40e_pf *pf = vsi->back;
4178         int i, pf_q, ret = 0;
4179
4180         pf_q = vsi->base_queue;
4181         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4182                 i40e_control_rx_q(pf, pf_q, enable);
4183
4184                 /* wait for the change to finish */
4185                 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4186                 if (ret) {
4187                         dev_info(&pf->pdev->dev,
4188                                  "VSI seid %d Rx ring %d %sable timeout\n",
4189                                  vsi->seid, pf_q, (enable ? "en" : "dis"));
4190                         break;
4191                 }
4192         }
4193
4194         /* Due to HW errata, on Rx disable only, the register can indicate done
4195          * before it really is. Needs 50ms to be sure
4196          */
4197         if (!enable)
4198                 mdelay(50);
4199
4200         return ret;
4201 }
4202
4203 /**
4204  * i40e_vsi_start_rings - Start a VSI's rings
4205  * @vsi: the VSI being configured
4206  **/
4207 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4208 {
4209         int ret = 0;
4210
4211         /* do rx first for enable and last for disable */
4212         ret = i40e_vsi_control_rx(vsi, true);
4213         if (ret)
4214                 return ret;
4215         ret = i40e_vsi_control_tx(vsi, true);
4216
4217         return ret;
4218 }
4219
4220 /**
4221  * i40e_vsi_stop_rings - Stop a VSI's rings
4222  * @vsi: the VSI being configured
4223  **/
4224 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4225 {
4226         /* When port TX is suspended, don't wait */
4227         if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4228                 return i40e_vsi_stop_rings_no_wait(vsi);
4229
4230         /* do rx first for enable and last for disable
4231          * Ignore return value, we need to shutdown whatever we can
4232          */
4233         i40e_vsi_control_tx(vsi, false);
4234         i40e_vsi_control_rx(vsi, false);
4235 }
4236
4237 /**
4238  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4239  * @vsi: the VSI being shutdown
4240  *
4241  * This function stops all the rings for a VSI but does not delay to verify
4242  * that rings have been disabled. It is expected that the caller is shutting
4243  * down multiple VSIs at once and will delay together for all the VSIs after
4244  * initiating the shutdown. This is particularly useful for shutting down lots
4245  * of VFs together. Otherwise, a large delay can be incurred while configuring
4246  * each VSI in serial.
4247  **/
4248 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4249 {
4250         struct i40e_pf *pf = vsi->back;
4251         int i, pf_q;
4252
4253         pf_q = vsi->base_queue;
4254         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4255                 i40e_control_tx_q(pf, pf_q, false);
4256                 i40e_control_rx_q(pf, pf_q, false);
4257         }
4258 }
4259
4260 /**
4261  * i40e_vsi_free_irq - Free the irq association with the OS
4262  * @vsi: the VSI being configured
4263  **/
4264 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4265 {
4266         struct i40e_pf *pf = vsi->back;
4267         struct i40e_hw *hw = &pf->hw;
4268         int base = vsi->base_vector;
4269         u32 val, qp;
4270         int i;
4271
4272         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4273                 if (!vsi->q_vectors)
4274                         return;
4275
4276                 if (!vsi->irqs_ready)
4277                         return;
4278
4279                 vsi->irqs_ready = false;
4280                 for (i = 0; i < vsi->num_q_vectors; i++) {
4281                         int irq_num;
4282                         u16 vector;
4283
4284                         vector = i + base;
4285                         irq_num = pf->msix_entries[vector].vector;
4286
4287                         /* free only the irqs that were actually requested */
4288                         if (!vsi->q_vectors[i] ||
4289                             !vsi->q_vectors[i]->num_ringpairs)
4290                                 continue;
4291
4292                         /* clear the affinity notifier in the IRQ descriptor */
4293                         irq_set_affinity_notifier(irq_num, NULL);
4294                         /* remove our suggested affinity mask for this IRQ */
4295                         irq_set_affinity_hint(irq_num, NULL);
4296                         synchronize_irq(irq_num);
4297                         free_irq(irq_num, vsi->q_vectors[i]);
4298
4299                         /* Tear down the interrupt queue link list
4300                          *
4301                          * We know that they come in pairs and always
4302                          * the Rx first, then the Tx.  To clear the
4303                          * link list, stick the EOL value into the
4304                          * next_q field of the registers.
4305                          */
4306                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4307                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4308                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4309                         val |= I40E_QUEUE_END_OF_LIST
4310                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4311                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4312
4313                         while (qp != I40E_QUEUE_END_OF_LIST) {
4314                                 u32 next;
4315
4316                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4317
4318                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4319                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4320                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4321                                          I40E_QINT_RQCTL_INTEVENT_MASK);
4322
4323                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4324                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4325
4326                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4327
4328                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4329
4330                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4331                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4332
4333                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4334                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4335                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4336                                          I40E_QINT_TQCTL_INTEVENT_MASK);
4337
4338                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4339                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4340
4341                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4342                                 qp = next;
4343                         }
4344                 }
4345         } else {
4346                 free_irq(pf->pdev->irq, pf);
4347
4348                 val = rd32(hw, I40E_PFINT_LNKLST0);
4349                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4350                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4351                 val |= I40E_QUEUE_END_OF_LIST
4352                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4353                 wr32(hw, I40E_PFINT_LNKLST0, val);
4354
4355                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4356                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4357                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4358                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4359                          I40E_QINT_RQCTL_INTEVENT_MASK);
4360
4361                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4362                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4363
4364                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4365
4366                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4367
4368                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4369                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4370                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4371                          I40E_QINT_TQCTL_INTEVENT_MASK);
4372
4373                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4374                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4375
4376                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4377         }
4378 }
4379
4380 /**
4381  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4382  * @vsi: the VSI being configured
4383  * @v_idx: Index of vector to be freed
4384  *
4385  * This function frees the memory allocated to the q_vector.  In addition if
4386  * NAPI is enabled it will delete any references to the NAPI struct prior
4387  * to freeing the q_vector.
4388  **/
4389 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4390 {
4391         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4392         struct i40e_ring *ring;
4393
4394         if (!q_vector)
4395                 return;
4396
4397         /* disassociate q_vector from rings */
4398         i40e_for_each_ring(ring, q_vector->tx)
4399                 ring->q_vector = NULL;
4400
4401         i40e_for_each_ring(ring, q_vector->rx)
4402                 ring->q_vector = NULL;
4403
4404         /* only VSI w/ an associated netdev is set up w/ NAPI */
4405         if (vsi->netdev)
4406                 netif_napi_del(&q_vector->napi);
4407
4408         vsi->q_vectors[v_idx] = NULL;
4409
4410         kfree_rcu(q_vector, rcu);
4411 }
4412
4413 /**
4414  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4415  * @vsi: the VSI being un-configured
4416  *
4417  * This frees the memory allocated to the q_vectors and
4418  * deletes references to the NAPI struct.
4419  **/
4420 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4421 {
4422         int v_idx;
4423
4424         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4425                 i40e_free_q_vector(vsi, v_idx);
4426 }
4427
4428 /**
4429  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4430  * @pf: board private structure
4431  **/
4432 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4433 {
4434         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4435         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4436                 pci_disable_msix(pf->pdev);
4437                 kfree(pf->msix_entries);
4438                 pf->msix_entries = NULL;
4439                 kfree(pf->irq_pile);
4440                 pf->irq_pile = NULL;
4441         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4442                 pci_disable_msi(pf->pdev);
4443         }
4444         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4445 }
4446
4447 /**
4448  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4449  * @pf: board private structure
4450  *
4451  * We go through and clear interrupt specific resources and reset the structure
4452  * to pre-load conditions
4453  **/
4454 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4455 {
4456         int i;
4457
4458         i40e_stop_misc_vector(pf);
4459         if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
4460                 synchronize_irq(pf->msix_entries[0].vector);
4461                 free_irq(pf->msix_entries[0].vector, pf);
4462         }
4463
4464         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4465                       I40E_IWARP_IRQ_PILE_ID);
4466
4467         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4468         for (i = 0; i < pf->num_alloc_vsi; i++)
4469                 if (pf->vsi[i])
4470                         i40e_vsi_free_q_vectors(pf->vsi[i]);
4471         i40e_reset_interrupt_capability(pf);
4472 }
4473
4474 /**
4475  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4476  * @vsi: the VSI being configured
4477  **/
4478 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4479 {
4480         int q_idx;
4481
4482         if (!vsi->netdev)
4483                 return;
4484
4485         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4486                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4487
4488                 if (q_vector->rx.ring || q_vector->tx.ring)
4489                         napi_enable(&q_vector->napi);
4490         }
4491 }
4492
4493 /**
4494  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4495  * @vsi: the VSI being configured
4496  **/
4497 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4498 {
4499         int q_idx;
4500
4501         if (!vsi->netdev)
4502                 return;
4503
4504         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4505                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4506
4507                 if (q_vector->rx.ring || q_vector->tx.ring)
4508                         napi_disable(&q_vector->napi);
4509         }
4510 }
4511
4512 /**
4513  * i40e_vsi_close - Shut down a VSI
4514  * @vsi: the vsi to be quelled
4515  **/
4516 static void i40e_vsi_close(struct i40e_vsi *vsi)
4517 {
4518         struct i40e_pf *pf = vsi->back;
4519         if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
4520                 i40e_down(vsi);
4521         i40e_vsi_free_irq(vsi);
4522         i40e_vsi_free_tx_resources(vsi);
4523         i40e_vsi_free_rx_resources(vsi);
4524         vsi->current_netdev_flags = 0;
4525         pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
4526         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4527                 pf->flags |=  I40E_FLAG_CLIENT_RESET;
4528 }
4529
4530 /**
4531  * i40e_quiesce_vsi - Pause a given VSI
4532  * @vsi: the VSI being paused
4533  **/
4534 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4535 {
4536         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4537                 return;
4538
4539         set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
4540         if (vsi->netdev && netif_running(vsi->netdev))
4541                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4542         else
4543                 i40e_vsi_close(vsi);
4544 }
4545
4546 /**
4547  * i40e_unquiesce_vsi - Resume a given VSI
4548  * @vsi: the VSI being resumed
4549  **/
4550 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
4551 {
4552         if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
4553                 return;
4554
4555         if (vsi->netdev && netif_running(vsi->netdev))
4556                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
4557         else
4558                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
4559 }
4560
4561 /**
4562  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
4563  * @pf: the PF
4564  **/
4565 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
4566 {
4567         int v;
4568
4569         for (v = 0; v < pf->num_alloc_vsi; v++) {
4570                 if (pf->vsi[v])
4571                         i40e_quiesce_vsi(pf->vsi[v]);
4572         }
4573 }
4574
4575 /**
4576  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
4577  * @pf: the PF
4578  **/
4579 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
4580 {
4581         int v;
4582
4583         for (v = 0; v < pf->num_alloc_vsi; v++) {
4584                 if (pf->vsi[v])
4585                         i40e_unquiesce_vsi(pf->vsi[v]);
4586         }
4587 }
4588
4589 /**
4590  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
4591  * @vsi: the VSI being configured
4592  *
4593  * Wait until all queues on a given VSI have been disabled.
4594  **/
4595 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
4596 {
4597         struct i40e_pf *pf = vsi->back;
4598         int i, pf_q, ret;
4599
4600         pf_q = vsi->base_queue;
4601         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4602                 /* Check and wait for the Tx queue */
4603                 ret = i40e_pf_txq_wait(pf, pf_q, false);
4604                 if (ret) {
4605                         dev_info(&pf->pdev->dev,
4606                                  "VSI seid %d Tx ring %d disable timeout\n",
4607                                  vsi->seid, pf_q);
4608                         return ret;
4609                 }
4610
4611                 if (!i40e_enabled_xdp_vsi(vsi))
4612                         goto wait_rx;
4613
4614                 /* Check and wait for the XDP Tx queue */
4615                 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
4616                                        false);
4617                 if (ret) {
4618                         dev_info(&pf->pdev->dev,
4619                                  "VSI seid %d XDP Tx ring %d disable timeout\n",
4620                                  vsi->seid, pf_q);
4621                         return ret;
4622                 }
4623 wait_rx:
4624                 /* Check and wait for the Rx queue */
4625                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
4626                 if (ret) {
4627                         dev_info(&pf->pdev->dev,
4628                                  "VSI seid %d Rx ring %d disable timeout\n",
4629                                  vsi->seid, pf_q);
4630                         return ret;
4631                 }
4632         }
4633
4634         return 0;
4635 }
4636
4637 #ifdef CONFIG_I40E_DCB
4638 /**
4639  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
4640  * @pf: the PF
4641  *
4642  * This function waits for the queues to be in disabled state for all the
4643  * VSIs that are managed by this PF.
4644  **/
4645 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
4646 {
4647         int v, ret = 0;
4648
4649         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4650                 if (pf->vsi[v]) {
4651                         ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
4652                         if (ret)
4653                                 break;
4654                 }
4655         }
4656
4657         return ret;
4658 }
4659
4660 #endif
4661
4662 /**
4663  * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue
4664  * @q_idx: TX queue number
4665  * @vsi: Pointer to VSI struct
4666  *
4667  * This function checks specified queue for given VSI. Detects hung condition.
4668  * We proactively detect hung TX queues by checking if interrupts are disabled
4669  * but there are pending descriptors.  If it appears hung, attempt to recover
4670  * by triggering a SW interrupt.
4671  **/
4672 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi)
4673 {
4674         struct i40e_ring *tx_ring = NULL;
4675         struct i40e_pf  *pf;
4676         u32 val, tx_pending;
4677         int i;
4678
4679         pf = vsi->back;
4680
4681         /* now that we have an index, find the tx_ring struct */
4682         for (i = 0; i < vsi->num_queue_pairs; i++) {
4683                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
4684                         if (q_idx == vsi->tx_rings[i]->queue_index) {
4685                                 tx_ring = vsi->tx_rings[i];
4686                                 break;
4687                         }
4688                 }
4689         }
4690
4691         if (!tx_ring)
4692                 return;
4693
4694         /* Read interrupt register */
4695         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4696                 val = rd32(&pf->hw,
4697                            I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
4698                                                tx_ring->vsi->base_vector - 1));
4699         else
4700                 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
4701
4702         tx_pending = i40e_get_tx_pending(tx_ring);
4703
4704         /* Interrupts are disabled and TX pending is non-zero,
4705          * trigger the SW interrupt (don't wait). Worst case
4706          * there will be one extra interrupt which may result
4707          * into not cleaning any queues because queues are cleaned.
4708          */
4709         if (tx_pending && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK)))
4710                 i40e_force_wb(vsi, tx_ring->q_vector);
4711 }
4712
4713 /**
4714  * i40e_detect_recover_hung - Function to detect and recover hung_queues
4715  * @pf:  pointer to PF struct
4716  *
4717  * LAN VSI has netdev and netdev has TX queues. This function is to check
4718  * each of those TX queues if they are hung, trigger recovery by issuing
4719  * SW interrupt.
4720  **/
4721 static void i40e_detect_recover_hung(struct i40e_pf *pf)
4722 {
4723         struct net_device *netdev;
4724         struct i40e_vsi *vsi;
4725         unsigned int i;
4726
4727         /* Only for LAN VSI */
4728         vsi = pf->vsi[pf->lan_vsi];
4729
4730         if (!vsi)
4731                 return;
4732
4733         /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */
4734         if (test_bit(__I40E_VSI_DOWN, vsi->back->state) ||
4735             test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state))
4736                 return;
4737
4738         /* Make sure type is MAIN VSI */
4739         if (vsi->type != I40E_VSI_MAIN)
4740                 return;
4741
4742         netdev = vsi->netdev;
4743         if (!netdev)
4744                 return;
4745
4746         /* Bail out if netif_carrier is not OK */
4747         if (!netif_carrier_ok(netdev))
4748                 return;
4749
4750         /* Go thru' TX queues for netdev */
4751         for (i = 0; i < netdev->num_tx_queues; i++) {
4752                 struct netdev_queue *q;
4753
4754                 q = netdev_get_tx_queue(netdev, i);
4755                 if (q)
4756                         i40e_detect_recover_hung_queue(i, vsi);
4757         }
4758 }
4759
4760 /**
4761  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
4762  * @pf: pointer to PF
4763  *
4764  * Get TC map for ISCSI PF type that will include iSCSI TC
4765  * and LAN TC.
4766  **/
4767 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
4768 {
4769         struct i40e_dcb_app_priority_table app;
4770         struct i40e_hw *hw = &pf->hw;
4771         u8 enabled_tc = 1; /* TC0 is always enabled */
4772         u8 tc, i;
4773         /* Get the iSCSI APP TLV */
4774         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4775
4776         for (i = 0; i < dcbcfg->numapps; i++) {
4777                 app = dcbcfg->app[i];
4778                 if (app.selector == I40E_APP_SEL_TCPIP &&
4779                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
4780                         tc = dcbcfg->etscfg.prioritytable[app.priority];
4781                         enabled_tc |= BIT(tc);
4782                         break;
4783                 }
4784         }
4785
4786         return enabled_tc;
4787 }
4788
4789 /**
4790  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
4791  * @dcbcfg: the corresponding DCBx configuration structure
4792  *
4793  * Return the number of TCs from given DCBx configuration
4794  **/
4795 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
4796 {
4797         int i, tc_unused = 0;
4798         u8 num_tc = 0;
4799         u8 ret = 0;
4800
4801         /* Scan the ETS Config Priority Table to find
4802          * traffic class enabled for a given priority
4803          * and create a bitmask of enabled TCs
4804          */
4805         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
4806                 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
4807
4808         /* Now scan the bitmask to check for
4809          * contiguous TCs starting with TC0
4810          */
4811         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4812                 if (num_tc & BIT(i)) {
4813                         if (!tc_unused) {
4814                                 ret++;
4815                         } else {
4816                                 pr_err("Non-contiguous TC - Disabling DCB\n");
4817                                 return 1;
4818                         }
4819                 } else {
4820                         tc_unused = 1;
4821                 }
4822         }
4823
4824         /* There is always at least TC0 */
4825         if (!ret)
4826                 ret = 1;
4827
4828         return ret;
4829 }
4830
4831 /**
4832  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
4833  * @dcbcfg: the corresponding DCBx configuration structure
4834  *
4835  * Query the current DCB configuration and return the number of
4836  * traffic classes enabled from the given DCBX config
4837  **/
4838 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
4839 {
4840         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
4841         u8 enabled_tc = 1;
4842         u8 i;
4843
4844         for (i = 0; i < num_tc; i++)
4845                 enabled_tc |= BIT(i);
4846
4847         return enabled_tc;
4848 }
4849
4850 /**
4851  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
4852  * @pf: PF being queried
4853  *
4854  * Return number of traffic classes enabled for the given PF
4855  **/
4856 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
4857 {
4858         struct i40e_hw *hw = &pf->hw;
4859         u8 i, enabled_tc = 1;
4860         u8 num_tc = 0;
4861         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4862
4863         /* If DCB is not enabled then always in single TC */
4864         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4865                 return 1;
4866
4867         /* SFP mode will be enabled for all TCs on port */
4868         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4869                 return i40e_dcb_get_num_tc(dcbcfg);
4870
4871         /* MFP mode return count of enabled TCs for this PF */
4872         if (pf->hw.func_caps.iscsi)
4873                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
4874         else
4875                 return 1; /* Only TC0 */
4876
4877         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4878                 if (enabled_tc & BIT(i))
4879                         num_tc++;
4880         }
4881         return num_tc;
4882 }
4883
4884 /**
4885  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
4886  * @pf: PF being queried
4887  *
4888  * Return a bitmap for enabled traffic classes for this PF.
4889  **/
4890 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
4891 {
4892         /* If DCB is not enabled for this PF then just return default TC */
4893         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4894                 return I40E_DEFAULT_TRAFFIC_CLASS;
4895
4896         /* SFP mode we want PF to be enabled for all TCs */
4897         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4898                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
4899
4900         /* MFP enabled and iSCSI PF type */
4901         if (pf->hw.func_caps.iscsi)
4902                 return i40e_get_iscsi_tc_map(pf);
4903         else
4904                 return I40E_DEFAULT_TRAFFIC_CLASS;
4905 }
4906
4907 /**
4908  * i40e_vsi_get_bw_info - Query VSI BW Information
4909  * @vsi: the VSI being queried
4910  *
4911  * Returns 0 on success, negative value on failure
4912  **/
4913 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
4914 {
4915         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
4916         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
4917         struct i40e_pf *pf = vsi->back;
4918         struct i40e_hw *hw = &pf->hw;
4919         i40e_status ret;
4920         u32 tc_bw_max;
4921         int i;
4922
4923         /* Get the VSI level BW configuration */
4924         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
4925         if (ret) {
4926                 dev_info(&pf->pdev->dev,
4927                          "couldn't get PF vsi bw config, err %s aq_err %s\n",
4928                          i40e_stat_str(&pf->hw, ret),
4929                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4930                 return -EINVAL;
4931         }
4932
4933         /* Get the VSI level BW configuration per TC */
4934         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
4935                                                NULL);
4936         if (ret) {
4937                 dev_info(&pf->pdev->dev,
4938                          "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
4939                          i40e_stat_str(&pf->hw, ret),
4940                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4941                 return -EINVAL;
4942         }
4943
4944         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
4945                 dev_info(&pf->pdev->dev,
4946                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
4947                          bw_config.tc_valid_bits,
4948                          bw_ets_config.tc_valid_bits);
4949                 /* Still continuing */
4950         }
4951
4952         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
4953         vsi->bw_max_quanta = bw_config.max_bw;
4954         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
4955                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
4956         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4957                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
4958                 vsi->bw_ets_limit_credits[i] =
4959                                         le16_to_cpu(bw_ets_config.credits[i]);
4960                 /* 3 bits out of 4 for each TC */
4961                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
4962         }
4963
4964         return 0;
4965 }
4966
4967 /**
4968  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
4969  * @vsi: the VSI being configured
4970  * @enabled_tc: TC bitmap
4971  * @bw_credits: BW shared credits per TC
4972  *
4973  * Returns 0 on success, negative value on failure
4974  **/
4975 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
4976                                        u8 *bw_share)
4977 {
4978         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
4979         i40e_status ret;
4980         int i;
4981
4982         bw_data.tc_valid_bits = enabled_tc;
4983         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4984                 bw_data.tc_bw_credits[i] = bw_share[i];
4985
4986         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
4987                                        NULL);
4988         if (ret) {
4989                 dev_info(&vsi->back->pdev->dev,
4990                          "AQ command Config VSI BW allocation per TC failed = %d\n",
4991                          vsi->back->hw.aq.asq_last_status);
4992                 return -EINVAL;
4993         }
4994
4995         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4996                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
4997
4998         return 0;
4999 }
5000
5001 /**
5002  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5003  * @vsi: the VSI being configured
5004  * @enabled_tc: TC map to be enabled
5005  *
5006  **/
5007 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5008 {
5009         struct net_device *netdev = vsi->netdev;
5010         struct i40e_pf *pf = vsi->back;
5011         struct i40e_hw *hw = &pf->hw;
5012         u8 netdev_tc = 0;
5013         int i;
5014         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5015
5016         if (!netdev)
5017                 return;
5018
5019         if (!enabled_tc) {
5020                 netdev_reset_tc(netdev);
5021                 return;
5022         }
5023
5024         /* Set up actual enabled TCs on the VSI */
5025         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5026                 return;
5027
5028         /* set per TC queues for the VSI */
5029         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5030                 /* Only set TC queues for enabled tcs
5031                  *
5032                  * e.g. For a VSI that has TC0 and TC3 enabled the
5033                  * enabled_tc bitmap would be 0x00001001; the driver
5034                  * will set the numtc for netdev as 2 that will be
5035                  * referenced by the netdev layer as TC 0 and 1.
5036                  */
5037                 if (vsi->tc_config.enabled_tc & BIT(i))
5038                         netdev_set_tc_queue(netdev,
5039                                         vsi->tc_config.tc_info[i].netdev_tc,
5040                                         vsi->tc_config.tc_info[i].qcount,
5041                                         vsi->tc_config.tc_info[i].qoffset);
5042         }
5043
5044         /* Assign UP2TC map for the VSI */
5045         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5046                 /* Get the actual TC# for the UP */
5047                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5048                 /* Get the mapped netdev TC# for the UP */
5049                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
5050                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5051         }
5052 }
5053
5054 /**
5055  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5056  * @vsi: the VSI being configured
5057  * @ctxt: the ctxt buffer returned from AQ VSI update param command
5058  **/
5059 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5060                                       struct i40e_vsi_context *ctxt)
5061 {
5062         /* copy just the sections touched not the entire info
5063          * since not all sections are valid as returned by
5064          * update vsi params
5065          */
5066         vsi->info.mapping_flags = ctxt->info.mapping_flags;
5067         memcpy(&vsi->info.queue_mapping,
5068                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5069         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5070                sizeof(vsi->info.tc_mapping));
5071 }
5072
5073 /**
5074  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5075  * @vsi: VSI to be configured
5076  * @enabled_tc: TC bitmap
5077  *
5078  * This configures a particular VSI for TCs that are mapped to the
5079  * given TC bitmap. It uses default bandwidth share for TCs across
5080  * VSIs to configure TC for a particular VSI.
5081  *
5082  * NOTE:
5083  * It is expected that the VSI queues have been quisced before calling
5084  * this function.
5085  **/
5086 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5087 {
5088         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5089         struct i40e_vsi_context ctxt;
5090         int ret = 0;
5091         int i;
5092
5093         /* Check if enabled_tc is same as existing or new TCs */
5094         if (vsi->tc_config.enabled_tc == enabled_tc)
5095                 return ret;
5096
5097         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5098         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5099                 if (enabled_tc & BIT(i))
5100                         bw_share[i] = 1;
5101         }
5102
5103         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5104         if (ret) {
5105                 dev_info(&vsi->back->pdev->dev,
5106                          "Failed configuring TC map %d for VSI %d\n",
5107                          enabled_tc, vsi->seid);
5108                 goto out;
5109         }
5110
5111         /* Update Queue Pairs Mapping for currently enabled UPs */
5112         ctxt.seid = vsi->seid;
5113         ctxt.pf_num = vsi->back->hw.pf_id;
5114         ctxt.vf_num = 0;
5115         ctxt.uplink_seid = vsi->uplink_seid;
5116         ctxt.info = vsi->info;
5117         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5118
5119         if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5120                 ctxt.info.valid_sections |=
5121                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5122                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5123         }
5124
5125         /* Update the VSI after updating the VSI queue-mapping information */
5126         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
5127         if (ret) {
5128                 dev_info(&vsi->back->pdev->dev,
5129                          "Update vsi tc config failed, err %s aq_err %s\n",
5130                          i40e_stat_str(&vsi->back->hw, ret),
5131                          i40e_aq_str(&vsi->back->hw,
5132                                      vsi->back->hw.aq.asq_last_status));
5133                 goto out;
5134         }
5135         /* update the local VSI info with updated queue map */
5136         i40e_vsi_update_queue_map(vsi, &ctxt);
5137         vsi->info.valid_sections = 0;
5138
5139         /* Update current VSI BW information */
5140         ret = i40e_vsi_get_bw_info(vsi);
5141         if (ret) {
5142                 dev_info(&vsi->back->pdev->dev,
5143                          "Failed updating vsi bw info, err %s aq_err %s\n",
5144                          i40e_stat_str(&vsi->back->hw, ret),
5145                          i40e_aq_str(&vsi->back->hw,
5146                                      vsi->back->hw.aq.asq_last_status));
5147                 goto out;
5148         }
5149
5150         /* Update the netdev TC setup */
5151         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5152 out:
5153         return ret;
5154 }
5155
5156 /**
5157  * i40e_veb_config_tc - Configure TCs for given VEB
5158  * @veb: given VEB
5159  * @enabled_tc: TC bitmap
5160  *
5161  * Configures given TC bitmap for VEB (switching) element
5162  **/
5163 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
5164 {
5165         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
5166         struct i40e_pf *pf = veb->pf;
5167         int ret = 0;
5168         int i;
5169
5170         /* No TCs or already enabled TCs just return */
5171         if (!enabled_tc || veb->enabled_tc == enabled_tc)
5172                 return ret;
5173
5174         bw_data.tc_valid_bits = enabled_tc;
5175         /* bw_data.absolute_credits is not set (relative) */
5176
5177         /* Enable ETS TCs with equal BW Share for now */
5178         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5179                 if (enabled_tc & BIT(i))
5180                         bw_data.tc_bw_share_credits[i] = 1;
5181         }
5182
5183         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
5184                                                    &bw_data, NULL);
5185         if (ret) {
5186                 dev_info(&pf->pdev->dev,
5187                          "VEB bw config failed, err %s aq_err %s\n",
5188                          i40e_stat_str(&pf->hw, ret),
5189                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5190                 goto out;
5191         }
5192
5193         /* Update the BW information */
5194         ret = i40e_veb_get_bw_info(veb);
5195         if (ret) {
5196                 dev_info(&pf->pdev->dev,
5197                          "Failed getting veb bw config, err %s aq_err %s\n",
5198                          i40e_stat_str(&pf->hw, ret),
5199                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5200         }
5201
5202 out:
5203         return ret;
5204 }
5205
5206 #ifdef CONFIG_I40E_DCB
5207 /**
5208  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
5209  * @pf: PF struct
5210  *
5211  * Reconfigure VEB/VSIs on a given PF; it is assumed that
5212  * the caller would've quiesce all the VSIs before calling
5213  * this function
5214  **/
5215 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
5216 {
5217         u8 tc_map = 0;
5218         int ret;
5219         u8 v;
5220
5221         /* Enable the TCs available on PF to all VEBs */
5222         tc_map = i40e_pf_get_tc_map(pf);
5223         for (v = 0; v < I40E_MAX_VEB; v++) {
5224                 if (!pf->veb[v])
5225                         continue;
5226                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
5227                 if (ret) {
5228                         dev_info(&pf->pdev->dev,
5229                                  "Failed configuring TC for VEB seid=%d\n",
5230                                  pf->veb[v]->seid);
5231                         /* Will try to configure as many components */
5232                 }
5233         }
5234
5235         /* Update each VSI */
5236         for (v = 0; v < pf->num_alloc_vsi; v++) {
5237                 if (!pf->vsi[v])
5238                         continue;
5239
5240                 /* - Enable all TCs for the LAN VSI
5241                  * - For all others keep them at TC0 for now
5242                  */
5243                 if (v == pf->lan_vsi)
5244                         tc_map = i40e_pf_get_tc_map(pf);
5245                 else
5246                         tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
5247
5248                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
5249                 if (ret) {
5250                         dev_info(&pf->pdev->dev,
5251                                  "Failed configuring TC for VSI seid=%d\n",
5252                                  pf->vsi[v]->seid);
5253                         /* Will try to configure as many components */
5254                 } else {
5255                         /* Re-configure VSI vectors based on updated TC map */
5256                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
5257                         if (pf->vsi[v]->netdev)
5258                                 i40e_dcbnl_set_all(pf->vsi[v]);
5259                 }
5260         }
5261 }
5262
5263 /**
5264  * i40e_resume_port_tx - Resume port Tx
5265  * @pf: PF struct
5266  *
5267  * Resume a port's Tx and issue a PF reset in case of failure to
5268  * resume.
5269  **/
5270 static int i40e_resume_port_tx(struct i40e_pf *pf)
5271 {
5272         struct i40e_hw *hw = &pf->hw;
5273         int ret;
5274
5275         ret = i40e_aq_resume_port_tx(hw, NULL);
5276         if (ret) {
5277                 dev_info(&pf->pdev->dev,
5278                          "Resume Port Tx failed, err %s aq_err %s\n",
5279                           i40e_stat_str(&pf->hw, ret),
5280                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5281                 /* Schedule PF reset to recover */
5282                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
5283                 i40e_service_event_schedule(pf);
5284         }
5285
5286         return ret;
5287 }
5288
5289 /**
5290  * i40e_init_pf_dcb - Initialize DCB configuration
5291  * @pf: PF being configured
5292  *
5293  * Query the current DCB configuration and cache it
5294  * in the hardware structure
5295  **/
5296 static int i40e_init_pf_dcb(struct i40e_pf *pf)
5297 {
5298         struct i40e_hw *hw = &pf->hw;
5299         int err = 0;
5300
5301         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */
5302         if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT)
5303                 goto out;
5304
5305         /* Get the initial DCB configuration */
5306         err = i40e_init_dcb(hw);
5307         if (!err) {
5308                 /* Device/Function is not DCBX capable */
5309                 if ((!hw->func_caps.dcb) ||
5310                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
5311                         dev_info(&pf->pdev->dev,
5312                                  "DCBX offload is not supported or is disabled for this PF.\n");
5313                 } else {
5314                         /* When status is not DISABLED then DCBX in FW */
5315                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
5316                                        DCB_CAP_DCBX_VER_IEEE;
5317
5318                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
5319                         /* Enable DCB tagging only when more than one TC
5320                          * or explicitly disable if only one TC
5321                          */
5322                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
5323                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
5324                         else
5325                                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
5326                         dev_dbg(&pf->pdev->dev,
5327                                 "DCBX offload is supported for this PF.\n");
5328                 }
5329         } else {
5330                 dev_info(&pf->pdev->dev,
5331                          "Query for DCB configuration failed, err %s aq_err %s\n",
5332                          i40e_stat_str(&pf->hw, err),
5333                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5334         }
5335
5336 out:
5337         return err;
5338 }
5339 #endif /* CONFIG_I40E_DCB */
5340 #define SPEED_SIZE 14
5341 #define FC_SIZE 8
5342 /**
5343  * i40e_print_link_message - print link up or down
5344  * @vsi: the VSI for which link needs a message
5345  */
5346 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
5347 {
5348         enum i40e_aq_link_speed new_speed;
5349         char *speed = "Unknown";
5350         char *fc = "Unknown";
5351         char *fec = "";
5352         char *req_fec = "";
5353         char *an = "";
5354
5355         new_speed = vsi->back->hw.phy.link_info.link_speed;
5356
5357         if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
5358                 return;
5359         vsi->current_isup = isup;
5360         vsi->current_speed = new_speed;
5361         if (!isup) {
5362                 netdev_info(vsi->netdev, "NIC Link is Down\n");
5363                 return;
5364         }
5365
5366         /* Warn user if link speed on NPAR enabled partition is not at
5367          * least 10GB
5368          */
5369         if (vsi->back->hw.func_caps.npar_enable &&
5370             (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
5371              vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
5372                 netdev_warn(vsi->netdev,
5373                             "The partition detected link speed that is less than 10Gbps\n");
5374
5375         switch (vsi->back->hw.phy.link_info.link_speed) {
5376         case I40E_LINK_SPEED_40GB:
5377                 speed = "40 G";
5378                 break;
5379         case I40E_LINK_SPEED_20GB:
5380                 speed = "20 G";
5381                 break;
5382         case I40E_LINK_SPEED_25GB:
5383                 speed = "25 G";
5384                 break;
5385         case I40E_LINK_SPEED_10GB:
5386                 speed = "10 G";
5387                 break;
5388         case I40E_LINK_SPEED_1GB:
5389                 speed = "1000 M";
5390                 break;
5391         case I40E_LINK_SPEED_100MB:
5392                 speed = "100 M";
5393                 break;
5394         default:
5395                 break;
5396         }
5397
5398         switch (vsi->back->hw.fc.current_mode) {
5399         case I40E_FC_FULL:
5400                 fc = "RX/TX";
5401                 break;
5402         case I40E_FC_TX_PAUSE:
5403                 fc = "TX";
5404                 break;
5405         case I40E_FC_RX_PAUSE:
5406                 fc = "RX";
5407                 break;
5408         default:
5409                 fc = "None";
5410                 break;
5411         }
5412
5413         if (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
5414                 req_fec = ", Requested FEC: None";
5415                 fec = ", FEC: None";
5416                 an = ", Autoneg: False";
5417
5418                 if (vsi->back->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
5419                         an = ", Autoneg: True";
5420
5421                 if (vsi->back->hw.phy.link_info.fec_info &
5422                     I40E_AQ_CONFIG_FEC_KR_ENA)
5423                         fec = ", FEC: CL74 FC-FEC/BASE-R";
5424                 else if (vsi->back->hw.phy.link_info.fec_info &
5425                          I40E_AQ_CONFIG_FEC_RS_ENA)
5426                         fec = ", FEC: CL108 RS-FEC";
5427
5428                 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
5429                  * both RS and FC are requested
5430                  */
5431                 if (vsi->back->hw.phy.link_info.req_fec_info &
5432                     (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
5433                         if (vsi->back->hw.phy.link_info.req_fec_info &
5434                             I40E_AQ_REQUEST_FEC_RS)
5435                                 req_fec = ", Requested FEC: CL108 RS-FEC";
5436                         else
5437                                 req_fec = ", Requested FEC: CL74 FC-FEC/BASE-R";
5438                 }
5439         }
5440
5441         netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s%s, Flow Control: %s\n",
5442                     speed, req_fec, fec, an, fc);
5443 }
5444
5445 /**
5446  * i40e_up_complete - Finish the last steps of bringing up a connection
5447  * @vsi: the VSI being configured
5448  **/
5449 static int i40e_up_complete(struct i40e_vsi *vsi)
5450 {
5451         struct i40e_pf *pf = vsi->back;
5452         int err;
5453
5454         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5455                 i40e_vsi_configure_msix(vsi);
5456         else
5457                 i40e_configure_msi_and_legacy(vsi);
5458
5459         /* start rings */
5460         err = i40e_vsi_start_rings(vsi);
5461         if (err)
5462                 return err;
5463
5464         clear_bit(__I40E_VSI_DOWN, vsi->state);
5465         i40e_napi_enable_all(vsi);
5466         i40e_vsi_enable_irq(vsi);
5467
5468         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
5469             (vsi->netdev)) {
5470                 i40e_print_link_message(vsi, true);
5471                 netif_tx_start_all_queues(vsi->netdev);
5472                 netif_carrier_on(vsi->netdev);
5473         } else if (vsi->netdev) {
5474                 i40e_print_link_message(vsi, false);
5475                 /* need to check for qualified module here*/
5476                 if ((pf->hw.phy.link_info.link_info &
5477                         I40E_AQ_MEDIA_AVAILABLE) &&
5478                     (!(pf->hw.phy.link_info.an_info &
5479                         I40E_AQ_QUALIFIED_MODULE)))
5480                         netdev_err(vsi->netdev,
5481                                    "the driver failed to link because an unqualified module was detected.");
5482         }
5483
5484         /* replay FDIR SB filters */
5485         if (vsi->type == I40E_VSI_FDIR) {
5486                 /* reset fd counters */
5487                 pf->fd_add_err = 0;
5488                 pf->fd_atr_cnt = 0;
5489                 i40e_fdir_filter_restore(vsi);
5490         }
5491
5492         /* On the next run of the service_task, notify any clients of the new
5493          * opened netdev
5494          */
5495         pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
5496         i40e_service_event_schedule(pf);
5497
5498         return 0;
5499 }
5500
5501 /**
5502  * i40e_vsi_reinit_locked - Reset the VSI
5503  * @vsi: the VSI being configured
5504  *
5505  * Rebuild the ring structs after some configuration
5506  * has changed, e.g. MTU size.
5507  **/
5508 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
5509 {
5510         struct i40e_pf *pf = vsi->back;
5511
5512         WARN_ON(in_interrupt());
5513         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
5514                 usleep_range(1000, 2000);
5515         i40e_down(vsi);
5516
5517         i40e_up(vsi);
5518         clear_bit(__I40E_CONFIG_BUSY, pf->state);
5519 }
5520
5521 /**
5522  * i40e_up - Bring the connection back up after being down
5523  * @vsi: the VSI being configured
5524  **/
5525 int i40e_up(struct i40e_vsi *vsi)
5526 {
5527         int err;
5528
5529         err = i40e_vsi_configure(vsi);
5530         if (!err)
5531                 err = i40e_up_complete(vsi);
5532
5533         return err;
5534 }
5535
5536 /**
5537  * i40e_down - Shutdown the connection processing
5538  * @vsi: the VSI being stopped
5539  **/
5540 void i40e_down(struct i40e_vsi *vsi)
5541 {
5542         int i;
5543
5544         /* It is assumed that the caller of this function
5545          * sets the vsi->state __I40E_VSI_DOWN bit.
5546          */
5547         if (vsi->netdev) {
5548                 netif_carrier_off(vsi->netdev);
5549                 netif_tx_disable(vsi->netdev);
5550         }
5551         i40e_vsi_disable_irq(vsi);
5552         i40e_vsi_stop_rings(vsi);
5553         i40e_napi_disable_all(vsi);
5554
5555         for (i = 0; i < vsi->num_queue_pairs; i++) {
5556                 i40e_clean_tx_ring(vsi->tx_rings[i]);
5557                 if (i40e_enabled_xdp_vsi(vsi))
5558                         i40e_clean_tx_ring(vsi->xdp_rings[i]);
5559                 i40e_clean_rx_ring(vsi->rx_rings[i]);
5560         }
5561
5562 }
5563
5564 /**
5565  * i40e_setup_tc - configure multiple traffic classes
5566  * @netdev: net device to configure
5567  * @tc: number of traffic classes to enable
5568  **/
5569 static int i40e_setup_tc(struct net_device *netdev, u8 tc)
5570 {
5571         struct i40e_netdev_priv *np = netdev_priv(netdev);
5572         struct i40e_vsi *vsi = np->vsi;
5573         struct i40e_pf *pf = vsi->back;
5574         u8 enabled_tc = 0;
5575         int ret = -EINVAL;
5576         int i;
5577
5578         /* Check if DCB enabled to continue */
5579         if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5580                 netdev_info(netdev, "DCB is not enabled for adapter\n");
5581                 goto exit;
5582         }
5583
5584         /* Check if MFP enabled */
5585         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
5586                 netdev_info(netdev, "Configuring TC not supported in MFP mode\n");
5587                 goto exit;
5588         }
5589
5590         /* Check whether tc count is within enabled limit */
5591         if (tc > i40e_pf_get_num_tc(pf)) {
5592                 netdev_info(netdev, "TC count greater than enabled on link for adapter\n");
5593                 goto exit;
5594         }
5595
5596         /* Generate TC map for number of tc requested */
5597         for (i = 0; i < tc; i++)
5598                 enabled_tc |= BIT(i);
5599
5600         /* Requesting same TC configuration as already enabled */
5601         if (enabled_tc == vsi->tc_config.enabled_tc)
5602                 return 0;
5603
5604         /* Quiesce VSI queues */
5605         i40e_quiesce_vsi(vsi);
5606
5607         /* Configure VSI for enabled TCs */
5608         ret = i40e_vsi_config_tc(vsi, enabled_tc);
5609         if (ret) {
5610                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
5611                             vsi->seid);
5612                 goto exit;
5613         }
5614
5615         /* Unquiesce VSI */
5616         i40e_unquiesce_vsi(vsi);
5617
5618 exit:
5619         return ret;
5620 }
5621
5622 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
5623                            void *type_data)
5624 {
5625         struct tc_mqprio_qopt *mqprio = type_data;
5626
5627         if (type != TC_SETUP_MQPRIO)
5628                 return -EOPNOTSUPP;
5629
5630         mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
5631
5632         return i40e_setup_tc(netdev, mqprio->num_tc);
5633 }
5634
5635 /**
5636  * i40e_open - Called when a network interface is made active
5637  * @netdev: network interface device structure
5638  *
5639  * The open entry point is called when a network interface is made
5640  * active by the system (IFF_UP).  At this point all resources needed
5641  * for transmit and receive operations are allocated, the interrupt
5642  * handler is registered with the OS, the netdev watchdog subtask is
5643  * enabled, and the stack is notified that the interface is ready.
5644  *
5645  * Returns 0 on success, negative value on failure
5646  **/
5647 int i40e_open(struct net_device *netdev)
5648 {
5649         struct i40e_netdev_priv *np = netdev_priv(netdev);
5650         struct i40e_vsi *vsi = np->vsi;
5651         struct i40e_pf *pf = vsi->back;
5652         int err;
5653
5654         /* disallow open during test or if eeprom is broken */
5655         if (test_bit(__I40E_TESTING, pf->state) ||
5656             test_bit(__I40E_BAD_EEPROM, pf->state))
5657                 return -EBUSY;
5658
5659         netif_carrier_off(netdev);
5660
5661         err = i40e_vsi_open(vsi);
5662         if (err)
5663                 return err;
5664
5665         /* configure global TSO hardware offload settings */
5666         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
5667                                                        TCP_FLAG_FIN) >> 16);
5668         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
5669                                                        TCP_FLAG_FIN |
5670                                                        TCP_FLAG_CWR) >> 16);
5671         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
5672
5673         udp_tunnel_get_rx_info(netdev);
5674
5675         return 0;
5676 }
5677
5678 /**
5679  * i40e_vsi_open -
5680  * @vsi: the VSI to open
5681  *
5682  * Finish initialization of the VSI.
5683  *
5684  * Returns 0 on success, negative value on failure
5685  *
5686  * Note: expects to be called while under rtnl_lock()
5687  **/
5688 int i40e_vsi_open(struct i40e_vsi *vsi)
5689 {
5690         struct i40e_pf *pf = vsi->back;
5691         char int_name[I40E_INT_NAME_STR_LEN];
5692         int err;
5693
5694         /* allocate descriptors */
5695         err = i40e_vsi_setup_tx_resources(vsi);
5696         if (err)
5697                 goto err_setup_tx;
5698         err = i40e_vsi_setup_rx_resources(vsi);
5699         if (err)
5700                 goto err_setup_rx;
5701
5702         err = i40e_vsi_configure(vsi);
5703         if (err)
5704                 goto err_setup_rx;
5705
5706         if (vsi->netdev) {
5707                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
5708                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
5709                 err = i40e_vsi_request_irq(vsi, int_name);
5710                 if (err)
5711                         goto err_setup_rx;
5712
5713                 /* Notify the stack of the actual queue counts. */
5714                 err = netif_set_real_num_tx_queues(vsi->netdev,
5715                                                    vsi->num_queue_pairs);
5716                 if (err)
5717                         goto err_set_queues;
5718
5719                 err = netif_set_real_num_rx_queues(vsi->netdev,
5720                                                    vsi->num_queue_pairs);
5721                 if (err)
5722                         goto err_set_queues;
5723
5724         } else if (vsi->type == I40E_VSI_FDIR) {
5725                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
5726                          dev_driver_string(&pf->pdev->dev),
5727                          dev_name(&pf->pdev->dev));
5728                 err = i40e_vsi_request_irq(vsi, int_name);
5729
5730         } else {
5731                 err = -EINVAL;
5732                 goto err_setup_rx;
5733         }
5734
5735         err = i40e_up_complete(vsi);
5736         if (err)
5737                 goto err_up_complete;
5738
5739         return 0;
5740
5741 err_up_complete:
5742         i40e_down(vsi);
5743 err_set_queues:
5744         i40e_vsi_free_irq(vsi);
5745 err_setup_rx:
5746         i40e_vsi_free_rx_resources(vsi);
5747 err_setup_tx:
5748         i40e_vsi_free_tx_resources(vsi);
5749         if (vsi == pf->vsi[pf->lan_vsi])
5750                 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true);
5751
5752         return err;
5753 }
5754
5755 /**
5756  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
5757  * @pf: Pointer to PF
5758  *
5759  * This function destroys the hlist where all the Flow Director
5760  * filters were saved.
5761  **/
5762 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
5763 {
5764         struct i40e_fdir_filter *filter;
5765         struct i40e_flex_pit *pit_entry, *tmp;
5766         struct hlist_node *node2;
5767
5768         hlist_for_each_entry_safe(filter, node2,
5769                                   &pf->fdir_filter_list, fdir_node) {
5770                 hlist_del(&filter->fdir_node);
5771                 kfree(filter);
5772         }
5773
5774         list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
5775                 list_del(&pit_entry->list);
5776                 kfree(pit_entry);
5777         }
5778         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
5779
5780         list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
5781                 list_del(&pit_entry->list);
5782                 kfree(pit_entry);
5783         }
5784         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
5785
5786         pf->fdir_pf_active_filters = 0;
5787         pf->fd_tcp4_filter_cnt = 0;
5788         pf->fd_udp4_filter_cnt = 0;
5789         pf->fd_sctp4_filter_cnt = 0;
5790         pf->fd_ip4_filter_cnt = 0;
5791
5792         /* Reprogram the default input set for TCP/IPv4 */
5793         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
5794                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5795                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5796
5797         /* Reprogram the default input set for UDP/IPv4 */
5798         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
5799                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5800                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5801
5802         /* Reprogram the default input set for SCTP/IPv4 */
5803         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
5804                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5805                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5806
5807         /* Reprogram the default input set for Other/IPv4 */
5808         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
5809                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
5810 }
5811
5812 /**
5813  * i40e_close - Disables a network interface
5814  * @netdev: network interface device structure
5815  *
5816  * The close entry point is called when an interface is de-activated
5817  * by the OS.  The hardware is still under the driver's control, but
5818  * this netdev interface is disabled.
5819  *
5820  * Returns 0, this is not allowed to fail
5821  **/
5822 int i40e_close(struct net_device *netdev)
5823 {
5824         struct i40e_netdev_priv *np = netdev_priv(netdev);
5825         struct i40e_vsi *vsi = np->vsi;
5826
5827         i40e_vsi_close(vsi);
5828
5829         return 0;
5830 }
5831
5832 /**
5833  * i40e_do_reset - Start a PF or Core Reset sequence
5834  * @pf: board private structure
5835  * @reset_flags: which reset is requested
5836  * @lock_acquired: indicates whether or not the lock has been acquired
5837  * before this function was called.
5838  *
5839  * The essential difference in resets is that the PF Reset
5840  * doesn't clear the packet buffers, doesn't reset the PE
5841  * firmware, and doesn't bother the other PFs on the chip.
5842  **/
5843 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
5844 {
5845         u32 val;
5846
5847         WARN_ON(in_interrupt());
5848
5849
5850         /* do the biggest reset indicated */
5851         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
5852
5853                 /* Request a Global Reset
5854                  *
5855                  * This will start the chip's countdown to the actual full
5856                  * chip reset event, and a warning interrupt to be sent
5857                  * to all PFs, including the requestor.  Our handler
5858                  * for the warning interrupt will deal with the shutdown
5859                  * and recovery of the switch setup.
5860                  */
5861                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
5862                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5863                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
5864                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5865
5866         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
5867
5868                 /* Request a Core Reset
5869                  *
5870                  * Same as Global Reset, except does *not* include the MAC/PHY
5871                  */
5872                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
5873                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5874                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
5875                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5876                 i40e_flush(&pf->hw);
5877
5878         } else if (reset_flags & BIT_ULL(__I40E_PF_RESET_REQUESTED)) {
5879
5880                 /* Request a PF Reset
5881                  *
5882                  * Resets only the PF-specific registers
5883                  *
5884                  * This goes directly to the tear-down and rebuild of
5885                  * the switch, since we need to do all the recovery as
5886                  * for the Core Reset.
5887                  */
5888                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
5889                 i40e_handle_reset_warning(pf, lock_acquired);
5890
5891         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
5892                 int v;
5893
5894                 /* Find the VSI(s) that requested a re-init */
5895                 dev_info(&pf->pdev->dev,
5896                          "VSI reinit requested\n");
5897                 for (v = 0; v < pf->num_alloc_vsi; v++) {
5898                         struct i40e_vsi *vsi = pf->vsi[v];
5899
5900                         if (vsi != NULL &&
5901                             test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
5902                                                vsi->state))
5903                                 i40e_vsi_reinit_locked(pf->vsi[v]);
5904                 }
5905         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
5906                 int v;
5907
5908                 /* Find the VSI(s) that needs to be brought down */
5909                 dev_info(&pf->pdev->dev, "VSI down requested\n");
5910                 for (v = 0; v < pf->num_alloc_vsi; v++) {
5911                         struct i40e_vsi *vsi = pf->vsi[v];
5912
5913                         if (vsi != NULL &&
5914                             test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
5915                                                vsi->state)) {
5916                                 set_bit(__I40E_VSI_DOWN, vsi->state);
5917                                 i40e_down(vsi);
5918                         }
5919                 }
5920         } else {
5921                 dev_info(&pf->pdev->dev,
5922                          "bad reset request 0x%08x\n", reset_flags);
5923         }
5924 }
5925
5926 #ifdef CONFIG_I40E_DCB
5927 /**
5928  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
5929  * @pf: board private structure
5930  * @old_cfg: current DCB config
5931  * @new_cfg: new DCB config
5932  **/
5933 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
5934                             struct i40e_dcbx_config *old_cfg,
5935                             struct i40e_dcbx_config *new_cfg)
5936 {
5937         bool need_reconfig = false;
5938
5939         /* Check if ETS configuration has changed */
5940         if (memcmp(&new_cfg->etscfg,
5941                    &old_cfg->etscfg,
5942                    sizeof(new_cfg->etscfg))) {
5943                 /* If Priority Table has changed reconfig is needed */
5944                 if (memcmp(&new_cfg->etscfg.prioritytable,
5945                            &old_cfg->etscfg.prioritytable,
5946                            sizeof(new_cfg->etscfg.prioritytable))) {
5947                         need_reconfig = true;
5948                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
5949                 }
5950
5951                 if (memcmp(&new_cfg->etscfg.tcbwtable,
5952                            &old_cfg->etscfg.tcbwtable,
5953                            sizeof(new_cfg->etscfg.tcbwtable)))
5954                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
5955
5956                 if (memcmp(&new_cfg->etscfg.tsatable,
5957                            &old_cfg->etscfg.tsatable,
5958                            sizeof(new_cfg->etscfg.tsatable)))
5959                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
5960         }
5961
5962         /* Check if PFC configuration has changed */
5963         if (memcmp(&new_cfg->pfc,
5964                    &old_cfg->pfc,
5965                    sizeof(new_cfg->pfc))) {
5966                 need_reconfig = true;
5967                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
5968         }
5969
5970         /* Check if APP Table has changed */
5971         if (memcmp(&new_cfg->app,
5972                    &old_cfg->app,
5973                    sizeof(new_cfg->app))) {
5974                 need_reconfig = true;
5975                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
5976         }
5977
5978         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
5979         return need_reconfig;
5980 }
5981
5982 /**
5983  * i40e_handle_lldp_event - Handle LLDP Change MIB event
5984  * @pf: board private structure
5985  * @e: event info posted on ARQ
5986  **/
5987 static int i40e_handle_lldp_event(struct i40e_pf *pf,
5988                                   struct i40e_arq_event_info *e)
5989 {
5990         struct i40e_aqc_lldp_get_mib *mib =
5991                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
5992         struct i40e_hw *hw = &pf->hw;
5993         struct i40e_dcbx_config tmp_dcbx_cfg;
5994         bool need_reconfig = false;
5995         int ret = 0;
5996         u8 type;
5997
5998         /* Not DCB capable or capability disabled */
5999         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
6000                 return ret;
6001
6002         /* Ignore if event is not for Nearest Bridge */
6003         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
6004                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
6005         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
6006         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
6007                 return ret;
6008
6009         /* Check MIB Type and return if event for Remote MIB update */
6010         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
6011         dev_dbg(&pf->pdev->dev,
6012                 "LLDP event mib type %s\n", type ? "remote" : "local");
6013         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
6014                 /* Update the remote cached instance and return */
6015                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
6016                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
6017                                 &hw->remote_dcbx_config);
6018                 goto exit;
6019         }
6020
6021         /* Store the old configuration */
6022         tmp_dcbx_cfg = hw->local_dcbx_config;
6023
6024         /* Reset the old DCBx configuration data */
6025         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
6026         /* Get updated DCBX data from firmware */
6027         ret = i40e_get_dcb_config(&pf->hw);
6028         if (ret) {
6029                 dev_info(&pf->pdev->dev,
6030                          "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
6031                          i40e_stat_str(&pf->hw, ret),
6032                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6033                 goto exit;
6034         }
6035
6036         /* No change detected in DCBX configs */
6037         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
6038                     sizeof(tmp_dcbx_cfg))) {
6039                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
6040                 goto exit;
6041         }
6042
6043         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
6044                                                &hw->local_dcbx_config);
6045
6046         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
6047
6048         if (!need_reconfig)
6049                 goto exit;
6050
6051         /* Enable DCB tagging only when more than one TC */
6052         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
6053                 pf->flags |= I40E_FLAG_DCB_ENABLED;
6054         else
6055                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6056
6057         set_bit(__I40E_PORT_SUSPENDED, pf->state);
6058         /* Reconfiguration needed quiesce all VSIs */
6059         i40e_pf_quiesce_all_vsi(pf);
6060
6061         /* Changes in configuration update VEB/VSI */
6062         i40e_dcb_reconfigure(pf);
6063
6064         ret = i40e_resume_port_tx(pf);
6065
6066         clear_bit(__I40E_PORT_SUSPENDED, pf->state);
6067         /* In case of error no point in resuming VSIs */
6068         if (ret)
6069                 goto exit;
6070
6071         /* Wait for the PF's queues to be disabled */
6072         ret = i40e_pf_wait_queues_disabled(pf);
6073         if (ret) {
6074                 /* Schedule PF reset to recover */
6075                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6076                 i40e_service_event_schedule(pf);
6077         } else {
6078                 i40e_pf_unquiesce_all_vsi(pf);
6079         pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED |
6080                       I40E_FLAG_CLIENT_L2_CHANGE);
6081         }
6082
6083 exit:
6084         return ret;
6085 }
6086 #endif /* CONFIG_I40E_DCB */
6087
6088 /**
6089  * i40e_do_reset_safe - Protected reset path for userland calls.
6090  * @pf: board private structure
6091  * @reset_flags: which reset is requested
6092  *
6093  **/
6094 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
6095 {
6096         rtnl_lock();
6097         i40e_do_reset(pf, reset_flags, true);
6098         rtnl_unlock();
6099 }
6100
6101 /**
6102  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
6103  * @pf: board private structure
6104  * @e: event info posted on ARQ
6105  *
6106  * Handler for LAN Queue Overflow Event generated by the firmware for PF
6107  * and VF queues
6108  **/
6109 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
6110                                            struct i40e_arq_event_info *e)
6111 {
6112         struct i40e_aqc_lan_overflow *data =
6113                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
6114         u32 queue = le32_to_cpu(data->prtdcb_rupto);
6115         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
6116         struct i40e_hw *hw = &pf->hw;
6117         struct i40e_vf *vf;
6118         u16 vf_id;
6119
6120         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
6121                 queue, qtx_ctl);
6122
6123         /* Queue belongs to VF, find the VF and issue VF reset */
6124         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
6125             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
6126                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
6127                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
6128                 vf_id -= hw->func_caps.vf_base_id;
6129                 vf = &pf->vf[vf_id];
6130                 i40e_vc_notify_vf_reset(vf);
6131                 /* Allow VF to process pending reset notification */
6132                 msleep(20);
6133                 i40e_reset_vf(vf, false);
6134         }
6135 }
6136
6137 /**
6138  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
6139  * @pf: board private structure
6140  **/
6141 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
6142 {
6143         u32 val, fcnt_prog;
6144
6145         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
6146         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
6147         return fcnt_prog;
6148 }
6149
6150 /**
6151  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
6152  * @pf: board private structure
6153  **/
6154 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
6155 {
6156         u32 val, fcnt_prog;
6157
6158         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
6159         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
6160                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
6161                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
6162         return fcnt_prog;
6163 }
6164
6165 /**
6166  * i40e_get_global_fd_count - Get total FD filters programmed on device
6167  * @pf: board private structure
6168  **/
6169 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
6170 {
6171         u32 val, fcnt_prog;
6172
6173         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
6174         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
6175                     ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
6176                      I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
6177         return fcnt_prog;
6178 }
6179
6180 /**
6181  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
6182  * @pf: board private structure
6183  **/
6184 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
6185 {
6186         struct i40e_fdir_filter *filter;
6187         u32 fcnt_prog, fcnt_avail;
6188         struct hlist_node *node;
6189
6190         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
6191                 return;
6192
6193         /* Check if we have enough room to re-enable FDir SB capability. */
6194         fcnt_prog = i40e_get_global_fd_count(pf);
6195         fcnt_avail = pf->fdir_pf_filter_count;
6196         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
6197             (pf->fd_add_err == 0) ||
6198             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) {
6199                 if (pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED) {
6200                         pf->flags &= ~I40E_FLAG_FD_SB_AUTO_DISABLED;
6201                         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
6202                             (I40E_DEBUG_FD & pf->hw.debug_mask))
6203                                 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
6204                 }
6205         }
6206
6207         /* We should wait for even more space before re-enabling ATR.
6208          * Additionally, we cannot enable ATR as long as we still have TCP SB
6209          * rules active.
6210          */
6211         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
6212             (pf->fd_tcp4_filter_cnt == 0)) {
6213                 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) {
6214                         pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
6215                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
6216                             (I40E_DEBUG_FD & pf->hw.debug_mask))
6217                                 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
6218                 }
6219         }
6220
6221         /* if hw had a problem adding a filter, delete it */
6222         if (pf->fd_inv > 0) {
6223                 hlist_for_each_entry_safe(filter, node,
6224                                           &pf->fdir_filter_list, fdir_node) {
6225                         if (filter->fd_id == pf->fd_inv) {
6226                                 hlist_del(&filter->fdir_node);
6227                                 kfree(filter);
6228                                 pf->fdir_pf_active_filters--;
6229                         }
6230                 }
6231         }
6232 }
6233
6234 #define I40E_MIN_FD_FLUSH_INTERVAL 10
6235 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
6236 /**
6237  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
6238  * @pf: board private structure
6239  **/
6240 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
6241 {
6242         unsigned long min_flush_time;
6243         int flush_wait_retry = 50;
6244         bool disable_atr = false;
6245         int fd_room;
6246         int reg;
6247
6248         if (!time_after(jiffies, pf->fd_flush_timestamp +
6249                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
6250                 return;
6251
6252         /* If the flush is happening too quick and we have mostly SB rules we
6253          * should not re-enable ATR for some time.
6254          */
6255         min_flush_time = pf->fd_flush_timestamp +
6256                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
6257         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
6258
6259         if (!(time_after(jiffies, min_flush_time)) &&
6260             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
6261                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
6262                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
6263                 disable_atr = true;
6264         }
6265
6266         pf->fd_flush_timestamp = jiffies;
6267         pf->flags |= I40E_FLAG_FD_ATR_AUTO_DISABLED;
6268         /* flush all filters */
6269         wr32(&pf->hw, I40E_PFQF_CTL_1,
6270              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
6271         i40e_flush(&pf->hw);
6272         pf->fd_flush_cnt++;
6273         pf->fd_add_err = 0;
6274         do {
6275                 /* Check FD flush status every 5-6msec */
6276                 usleep_range(5000, 6000);
6277                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
6278                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
6279                         break;
6280         } while (flush_wait_retry--);
6281         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
6282                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
6283         } else {
6284                 /* replay sideband filters */
6285                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
6286                 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
6287                         pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
6288                 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
6289                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
6290                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
6291         }
6292 }
6293
6294 /**
6295  * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
6296  * @pf: board private structure
6297  **/
6298 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
6299 {
6300         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
6301 }
6302
6303 /* We can see up to 256 filter programming desc in transit if the filters are
6304  * being applied really fast; before we see the first
6305  * filter miss error on Rx queue 0. Accumulating enough error messages before
6306  * reacting will make sure we don't cause flush too often.
6307  */
6308 #define I40E_MAX_FD_PROGRAM_ERROR 256
6309
6310 /**
6311  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
6312  * @pf: board private structure
6313  **/
6314 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
6315 {
6316
6317         /* if interface is down do nothing */
6318         if (test_bit(__I40E_DOWN, pf->state))
6319                 return;
6320
6321         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
6322                 i40e_fdir_flush_and_replay(pf);
6323
6324         i40e_fdir_check_and_reenable(pf);
6325
6326 }
6327
6328 /**
6329  * i40e_vsi_link_event - notify VSI of a link event
6330  * @vsi: vsi to be notified
6331  * @link_up: link up or down
6332  **/
6333 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
6334 {
6335         if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
6336                 return;
6337
6338         switch (vsi->type) {
6339         case I40E_VSI_MAIN:
6340                 if (!vsi->netdev || !vsi->netdev_registered)
6341                         break;
6342
6343                 if (link_up) {
6344                         netif_carrier_on(vsi->netdev);
6345                         netif_tx_wake_all_queues(vsi->netdev);
6346                 } else {
6347                         netif_carrier_off(vsi->netdev);
6348                         netif_tx_stop_all_queues(vsi->netdev);
6349                 }
6350                 break;
6351
6352         case I40E_VSI_SRIOV:
6353         case I40E_VSI_VMDQ2:
6354         case I40E_VSI_CTRL:
6355         case I40E_VSI_IWARP:
6356         case I40E_VSI_MIRROR:
6357         default:
6358                 /* there is no notification for other VSIs */
6359                 break;
6360         }
6361 }
6362
6363 /**
6364  * i40e_veb_link_event - notify elements on the veb of a link event
6365  * @veb: veb to be notified
6366  * @link_up: link up or down
6367  **/
6368 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
6369 {
6370         struct i40e_pf *pf;
6371         int i;
6372
6373         if (!veb || !veb->pf)
6374                 return;
6375         pf = veb->pf;
6376
6377         /* depth first... */
6378         for (i = 0; i < I40E_MAX_VEB; i++)
6379                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
6380                         i40e_veb_link_event(pf->veb[i], link_up);
6381
6382         /* ... now the local VSIs */
6383         for (i = 0; i < pf->num_alloc_vsi; i++)
6384                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
6385                         i40e_vsi_link_event(pf->vsi[i], link_up);
6386 }
6387
6388 /**
6389  * i40e_link_event - Update netif_carrier status
6390  * @pf: board private structure
6391  **/
6392 static void i40e_link_event(struct i40e_pf *pf)
6393 {
6394         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6395         u8 new_link_speed, old_link_speed;
6396         i40e_status status;
6397         bool new_link, old_link;
6398
6399         /* save off old link status information */
6400         pf->hw.phy.link_info_old = pf->hw.phy.link_info;
6401
6402         /* set this to force the get_link_status call to refresh state */
6403         pf->hw.phy.get_link_info = true;
6404
6405         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
6406
6407         status = i40e_get_link_status(&pf->hw, &new_link);
6408
6409         /* On success, disable temp link polling */
6410         if (status == I40E_SUCCESS) {
6411                 if (pf->flags & I40E_FLAG_TEMP_LINK_POLLING)
6412                         pf->flags &= ~I40E_FLAG_TEMP_LINK_POLLING;
6413         } else {
6414                 /* Enable link polling temporarily until i40e_get_link_status
6415                  * returns I40E_SUCCESS
6416                  */
6417                 pf->flags |= I40E_FLAG_TEMP_LINK_POLLING;
6418                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
6419                         status);
6420                 return;
6421         }
6422
6423         old_link_speed = pf->hw.phy.link_info_old.link_speed;
6424         new_link_speed = pf->hw.phy.link_info.link_speed;
6425
6426         if (new_link == old_link &&
6427             new_link_speed == old_link_speed &&
6428             (test_bit(__I40E_VSI_DOWN, vsi->state) ||
6429              new_link == netif_carrier_ok(vsi->netdev)))
6430                 return;
6431
6432         if (!test_bit(__I40E_VSI_DOWN, vsi->state))
6433                 i40e_print_link_message(vsi, new_link);
6434
6435         /* Notify the base of the switch tree connected to
6436          * the link.  Floating VEBs are not notified.
6437          */
6438         if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
6439                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
6440         else
6441                 i40e_vsi_link_event(vsi, new_link);
6442
6443         if (pf->vf)
6444                 i40e_vc_notify_link_state(pf);
6445
6446         if (pf->flags & I40E_FLAG_PTP)
6447                 i40e_ptp_set_increment(pf);
6448 }
6449
6450 /**
6451  * i40e_watchdog_subtask - periodic checks not using event driven response
6452  * @pf: board private structure
6453  **/
6454 static void i40e_watchdog_subtask(struct i40e_pf *pf)
6455 {
6456         int i;
6457
6458         /* if interface is down do nothing */
6459         if (test_bit(__I40E_DOWN, pf->state) ||
6460             test_bit(__I40E_CONFIG_BUSY, pf->state))
6461                 return;
6462
6463         /* make sure we don't do these things too often */
6464         if (time_before(jiffies, (pf->service_timer_previous +
6465                                   pf->service_timer_period)))
6466                 return;
6467         pf->service_timer_previous = jiffies;
6468
6469         if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
6470             (pf->flags & I40E_FLAG_TEMP_LINK_POLLING))
6471                 i40e_link_event(pf);
6472
6473         /* Update the stats for active netdevs so the network stack
6474          * can look at updated numbers whenever it cares to
6475          */
6476         for (i = 0; i < pf->num_alloc_vsi; i++)
6477                 if (pf->vsi[i] && pf->vsi[i]->netdev)
6478                         i40e_update_stats(pf->vsi[i]);
6479
6480         if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
6481                 /* Update the stats for the active switching components */
6482                 for (i = 0; i < I40E_MAX_VEB; i++)
6483                         if (pf->veb[i])
6484                                 i40e_update_veb_stats(pf->veb[i]);
6485         }
6486
6487         i40e_ptp_rx_hang(pf);
6488         i40e_ptp_tx_hang(pf);
6489 }
6490
6491 /**
6492  * i40e_reset_subtask - Set up for resetting the device and driver
6493  * @pf: board private structure
6494  **/
6495 static void i40e_reset_subtask(struct i40e_pf *pf)
6496 {
6497         u32 reset_flags = 0;
6498
6499         if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
6500                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
6501                 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
6502         }
6503         if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
6504                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
6505                 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6506         }
6507         if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
6508                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
6509                 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
6510         }
6511         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
6512                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
6513                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
6514         }
6515         if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
6516                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
6517                 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
6518         }
6519
6520         /* If there's a recovery already waiting, it takes
6521          * precedence before starting a new reset sequence.
6522          */
6523         if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
6524                 i40e_prep_for_reset(pf, false);
6525                 i40e_reset(pf);
6526                 i40e_rebuild(pf, false, false);
6527         }
6528
6529         /* If we're already down or resetting, just bail */
6530         if (reset_flags &&
6531             !test_bit(__I40E_DOWN, pf->state) &&
6532             !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
6533                 i40e_do_reset(pf, reset_flags, false);
6534         }
6535 }
6536
6537 /**
6538  * i40e_handle_link_event - Handle link event
6539  * @pf: board private structure
6540  * @e: event info posted on ARQ
6541  **/
6542 static void i40e_handle_link_event(struct i40e_pf *pf,
6543                                    struct i40e_arq_event_info *e)
6544 {
6545         struct i40e_aqc_get_link_status *status =
6546                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
6547
6548         /* Do a new status request to re-enable LSE reporting
6549          * and load new status information into the hw struct
6550          * This completely ignores any state information
6551          * in the ARQ event info, instead choosing to always
6552          * issue the AQ update link status command.
6553          */
6554         i40e_link_event(pf);
6555
6556         /* check for unqualified module, if link is down */
6557         if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
6558             (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
6559             (!(status->link_info & I40E_AQ_LINK_UP)))
6560                 dev_err(&pf->pdev->dev,
6561                         "The driver failed to link because an unqualified module was detected.\n");
6562 }
6563
6564 /**
6565  * i40e_clean_adminq_subtask - Clean the AdminQ rings
6566  * @pf: board private structure
6567  **/
6568 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
6569 {
6570         struct i40e_arq_event_info event;
6571         struct i40e_hw *hw = &pf->hw;
6572         u16 pending, i = 0;
6573         i40e_status ret;
6574         u16 opcode;
6575         u32 oldval;
6576         u32 val;
6577
6578         /* Do not run clean AQ when PF reset fails */
6579         if (test_bit(__I40E_RESET_FAILED, pf->state))
6580                 return;
6581
6582         /* check for error indications */
6583         val = rd32(&pf->hw, pf->hw.aq.arq.len);
6584         oldval = val;
6585         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
6586                 if (hw->debug_mask & I40E_DEBUG_AQ)
6587                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
6588                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
6589         }
6590         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
6591                 if (hw->debug_mask & I40E_DEBUG_AQ)
6592                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
6593                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
6594                 pf->arq_overflows++;
6595         }
6596         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
6597                 if (hw->debug_mask & I40E_DEBUG_AQ)
6598                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
6599                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
6600         }
6601         if (oldval != val)
6602                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
6603
6604         val = rd32(&pf->hw, pf->hw.aq.asq.len);
6605         oldval = val;
6606         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
6607                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6608                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
6609                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
6610         }
6611         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
6612                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6613                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
6614                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
6615         }
6616         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
6617                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6618                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
6619                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
6620         }
6621         if (oldval != val)
6622                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
6623
6624         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
6625         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
6626         if (!event.msg_buf)
6627                 return;
6628
6629         do {
6630                 ret = i40e_clean_arq_element(hw, &event, &pending);
6631                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
6632                         break;
6633                 else if (ret) {
6634                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
6635                         break;
6636                 }
6637
6638                 opcode = le16_to_cpu(event.desc.opcode);
6639                 switch (opcode) {
6640
6641                 case i40e_aqc_opc_get_link_status:
6642                         i40e_handle_link_event(pf, &event);
6643                         break;
6644                 case i40e_aqc_opc_send_msg_to_pf:
6645                         ret = i40e_vc_process_vf_msg(pf,
6646                                         le16_to_cpu(event.desc.retval),
6647                                         le32_to_cpu(event.desc.cookie_high),
6648                                         le32_to_cpu(event.desc.cookie_low),
6649                                         event.msg_buf,
6650                                         event.msg_len);
6651                         break;
6652                 case i40e_aqc_opc_lldp_update_mib:
6653                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
6654 #ifdef CONFIG_I40E_DCB
6655                         rtnl_lock();
6656                         ret = i40e_handle_lldp_event(pf, &event);
6657                         rtnl_unlock();
6658 #endif /* CONFIG_I40E_DCB */
6659                         break;
6660                 case i40e_aqc_opc_event_lan_overflow:
6661                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
6662                         i40e_handle_lan_overflow_event(pf, &event);
6663                         break;
6664                 case i40e_aqc_opc_send_msg_to_peer:
6665                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
6666                         break;
6667                 case i40e_aqc_opc_nvm_erase:
6668                 case i40e_aqc_opc_nvm_update:
6669                 case i40e_aqc_opc_oem_post_update:
6670                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
6671                                    "ARQ NVM operation 0x%04x completed\n",
6672                                    opcode);
6673                         break;
6674                 default:
6675                         dev_info(&pf->pdev->dev,
6676                                  "ARQ: Unknown event 0x%04x ignored\n",
6677                                  opcode);
6678                         break;
6679                 }
6680         } while (i++ < pf->adminq_work_limit);
6681
6682         if (i < pf->adminq_work_limit)
6683                 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
6684
6685         /* re-enable Admin queue interrupt cause */
6686         val = rd32(hw, I40E_PFINT_ICR0_ENA);
6687         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
6688         wr32(hw, I40E_PFINT_ICR0_ENA, val);
6689         i40e_flush(hw);
6690
6691         kfree(event.msg_buf);
6692 }
6693
6694 /**
6695  * i40e_verify_eeprom - make sure eeprom is good to use
6696  * @pf: board private structure
6697  **/
6698 static void i40e_verify_eeprom(struct i40e_pf *pf)
6699 {
6700         int err;
6701
6702         err = i40e_diag_eeprom_test(&pf->hw);
6703         if (err) {
6704                 /* retry in case of garbage read */
6705                 err = i40e_diag_eeprom_test(&pf->hw);
6706                 if (err) {
6707                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
6708                                  err);
6709                         set_bit(__I40E_BAD_EEPROM, pf->state);
6710                 }
6711         }
6712
6713         if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
6714                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
6715                 clear_bit(__I40E_BAD_EEPROM, pf->state);
6716         }
6717 }
6718
6719 /**
6720  * i40e_enable_pf_switch_lb
6721  * @pf: pointer to the PF structure
6722  *
6723  * enable switch loop back or die - no point in a return value
6724  **/
6725 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
6726 {
6727         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6728         struct i40e_vsi_context ctxt;
6729         int ret;
6730
6731         ctxt.seid = pf->main_vsi_seid;
6732         ctxt.pf_num = pf->hw.pf_id;
6733         ctxt.vf_num = 0;
6734         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6735         if (ret) {
6736                 dev_info(&pf->pdev->dev,
6737                          "couldn't get PF vsi config, err %s aq_err %s\n",
6738                          i40e_stat_str(&pf->hw, ret),
6739                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6740                 return;
6741         }
6742         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6743         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6744         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6745
6746         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6747         if (ret) {
6748                 dev_info(&pf->pdev->dev,
6749                          "update vsi switch failed, err %s aq_err %s\n",
6750                          i40e_stat_str(&pf->hw, ret),
6751                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6752         }
6753 }
6754
6755 /**
6756  * i40e_disable_pf_switch_lb
6757  * @pf: pointer to the PF structure
6758  *
6759  * disable switch loop back or die - no point in a return value
6760  **/
6761 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
6762 {
6763         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6764         struct i40e_vsi_context ctxt;
6765         int ret;
6766
6767         ctxt.seid = pf->main_vsi_seid;
6768         ctxt.pf_num = pf->hw.pf_id;
6769         ctxt.vf_num = 0;
6770         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6771         if (ret) {
6772                 dev_info(&pf->pdev->dev,
6773                          "couldn't get PF vsi config, err %s aq_err %s\n",
6774                          i40e_stat_str(&pf->hw, ret),
6775                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6776                 return;
6777         }
6778         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6779         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6780         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6781
6782         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6783         if (ret) {
6784                 dev_info(&pf->pdev->dev,
6785                          "update vsi switch failed, err %s aq_err %s\n",
6786                          i40e_stat_str(&pf->hw, ret),
6787                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6788         }
6789 }
6790
6791 /**
6792  * i40e_config_bridge_mode - Configure the HW bridge mode
6793  * @veb: pointer to the bridge instance
6794  *
6795  * Configure the loop back mode for the LAN VSI that is downlink to the
6796  * specified HW bridge instance. It is expected this function is called
6797  * when a new HW bridge is instantiated.
6798  **/
6799 static void i40e_config_bridge_mode(struct i40e_veb *veb)
6800 {
6801         struct i40e_pf *pf = veb->pf;
6802
6803         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
6804                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
6805                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
6806         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
6807                 i40e_disable_pf_switch_lb(pf);
6808         else
6809                 i40e_enable_pf_switch_lb(pf);
6810 }
6811
6812 /**
6813  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
6814  * @veb: pointer to the VEB instance
6815  *
6816  * This is a recursive function that first builds the attached VSIs then
6817  * recurses in to build the next layer of VEB.  We track the connections
6818  * through our own index numbers because the seid's from the HW could
6819  * change across the reset.
6820  **/
6821 static int i40e_reconstitute_veb(struct i40e_veb *veb)
6822 {
6823         struct i40e_vsi *ctl_vsi = NULL;
6824         struct i40e_pf *pf = veb->pf;
6825         int v, veb_idx;
6826         int ret;
6827
6828         /* build VSI that owns this VEB, temporarily attached to base VEB */
6829         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
6830                 if (pf->vsi[v] &&
6831                     pf->vsi[v]->veb_idx == veb->idx &&
6832                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
6833                         ctl_vsi = pf->vsi[v];
6834                         break;
6835                 }
6836         }
6837         if (!ctl_vsi) {
6838                 dev_info(&pf->pdev->dev,
6839                          "missing owner VSI for veb_idx %d\n", veb->idx);
6840                 ret = -ENOENT;
6841                 goto end_reconstitute;
6842         }
6843         if (ctl_vsi != pf->vsi[pf->lan_vsi])
6844                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6845         ret = i40e_add_vsi(ctl_vsi);
6846         if (ret) {
6847                 dev_info(&pf->pdev->dev,
6848                          "rebuild of veb_idx %d owner VSI failed: %d\n",
6849                          veb->idx, ret);
6850                 goto end_reconstitute;
6851         }
6852         i40e_vsi_reset_stats(ctl_vsi);
6853
6854         /* create the VEB in the switch and move the VSI onto the VEB */
6855         ret = i40e_add_veb(veb, ctl_vsi);
6856         if (ret)
6857                 goto end_reconstitute;
6858
6859         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
6860                 veb->bridge_mode = BRIDGE_MODE_VEB;
6861         else
6862                 veb->bridge_mode = BRIDGE_MODE_VEPA;
6863         i40e_config_bridge_mode(veb);
6864
6865         /* create the remaining VSIs attached to this VEB */
6866         for (v = 0; v < pf->num_alloc_vsi; v++) {
6867                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
6868                         continue;
6869
6870                 if (pf->vsi[v]->veb_idx == veb->idx) {
6871                         struct i40e_vsi *vsi = pf->vsi[v];
6872
6873                         vsi->uplink_seid = veb->seid;
6874                         ret = i40e_add_vsi(vsi);
6875                         if (ret) {
6876                                 dev_info(&pf->pdev->dev,
6877                                          "rebuild of vsi_idx %d failed: %d\n",
6878                                          v, ret);
6879                                 goto end_reconstitute;
6880                         }
6881                         i40e_vsi_reset_stats(vsi);
6882                 }
6883         }
6884
6885         /* create any VEBs attached to this VEB - RECURSION */
6886         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
6887                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
6888                         pf->veb[veb_idx]->uplink_seid = veb->seid;
6889                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
6890                         if (ret)
6891                                 break;
6892                 }
6893         }
6894
6895 end_reconstitute:
6896         return ret;
6897 }
6898
6899 /**
6900  * i40e_get_capabilities - get info about the HW
6901  * @pf: the PF struct
6902  **/
6903 static int i40e_get_capabilities(struct i40e_pf *pf)
6904 {
6905         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
6906         u16 data_size;
6907         int buf_len;
6908         int err;
6909
6910         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
6911         do {
6912                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
6913                 if (!cap_buf)
6914                         return -ENOMEM;
6915
6916                 /* this loads the data into the hw struct for us */
6917                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
6918                                             &data_size,
6919                                             i40e_aqc_opc_list_func_capabilities,
6920                                             NULL);
6921                 /* data loaded, buffer no longer needed */
6922                 kfree(cap_buf);
6923
6924                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
6925                         /* retry with a larger buffer */
6926                         buf_len = data_size;
6927                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
6928                         dev_info(&pf->pdev->dev,
6929                                  "capability discovery failed, err %s aq_err %s\n",
6930                                  i40e_stat_str(&pf->hw, err),
6931                                  i40e_aq_str(&pf->hw,
6932                                              pf->hw.aq.asq_last_status));
6933                         return -ENODEV;
6934                 }
6935         } while (err);
6936
6937         if (pf->hw.debug_mask & I40E_DEBUG_USER)
6938                 dev_info(&pf->pdev->dev,
6939                          "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
6940                          pf->hw.pf_id, pf->hw.func_caps.num_vfs,
6941                          pf->hw.func_caps.num_msix_vectors,
6942                          pf->hw.func_caps.num_msix_vectors_vf,
6943                          pf->hw.func_caps.fd_filters_guaranteed,
6944                          pf->hw.func_caps.fd_filters_best_effort,
6945                          pf->hw.func_caps.num_tx_qp,
6946                          pf->hw.func_caps.num_vsis);
6947
6948 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
6949                        + pf->hw.func_caps.num_vfs)
6950         if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) {
6951                 dev_info(&pf->pdev->dev,
6952                          "got num_vsis %d, setting num_vsis to %d\n",
6953                          pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
6954                 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
6955         }
6956
6957         return 0;
6958 }
6959
6960 static int i40e_vsi_clear(struct i40e_vsi *vsi);
6961
6962 /**
6963  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
6964  * @pf: board private structure
6965  **/
6966 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
6967 {
6968         struct i40e_vsi *vsi;
6969
6970         /* quick workaround for an NVM issue that leaves a critical register
6971          * uninitialized
6972          */
6973         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
6974                 static const u32 hkey[] = {
6975                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
6976                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
6977                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
6978                         0x95b3a76d};
6979                 int i;
6980
6981                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
6982                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
6983         }
6984
6985         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
6986                 return;
6987
6988         /* find existing VSI and see if it needs configuring */
6989         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
6990
6991         /* create a new VSI if none exists */
6992         if (!vsi) {
6993                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
6994                                      pf->vsi[pf->lan_vsi]->seid, 0);
6995                 if (!vsi) {
6996                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
6997                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
6998                         return;
6999                 }
7000         }
7001
7002         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
7003 }
7004
7005 /**
7006  * i40e_fdir_teardown - release the Flow Director resources
7007  * @pf: board private structure
7008  **/
7009 static void i40e_fdir_teardown(struct i40e_pf *pf)
7010 {
7011         struct i40e_vsi *vsi;
7012
7013         i40e_fdir_filter_exit(pf);
7014         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
7015         if (vsi)
7016                 i40e_vsi_release(vsi);
7017 }
7018
7019 /**
7020  * i40e_prep_for_reset - prep for the core to reset
7021  * @pf: board private structure
7022  * @lock_acquired: indicates whether or not the lock has been acquired
7023  * before this function was called.
7024  *
7025  * Close up the VFs and other things in prep for PF Reset.
7026   **/
7027 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired)
7028 {
7029         struct i40e_hw *hw = &pf->hw;
7030         i40e_status ret = 0;
7031         u32 v;
7032
7033         clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
7034         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
7035                 return;
7036         if (i40e_check_asq_alive(&pf->hw))
7037                 i40e_vc_notify_reset(pf);
7038
7039         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
7040
7041         /* quiesce the VSIs and their queues that are not already DOWN */
7042         /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */
7043         if (!lock_acquired)
7044                 rtnl_lock();
7045         i40e_pf_quiesce_all_vsi(pf);
7046         if (!lock_acquired)
7047                 rtnl_unlock();
7048
7049         for (v = 0; v < pf->num_alloc_vsi; v++) {
7050                 if (pf->vsi[v])
7051                         pf->vsi[v]->seid = 0;
7052         }
7053
7054         i40e_shutdown_adminq(&pf->hw);
7055
7056         /* call shutdown HMC */
7057         if (hw->hmc.hmc_obj) {
7058                 ret = i40e_shutdown_lan_hmc(hw);
7059                 if (ret)
7060                         dev_warn(&pf->pdev->dev,
7061                                  "shutdown_lan_hmc failed: %d\n", ret);
7062         }
7063 }
7064
7065 /**
7066  * i40e_send_version - update firmware with driver version
7067  * @pf: PF struct
7068  */
7069 static void i40e_send_version(struct i40e_pf *pf)
7070 {
7071         struct i40e_driver_version dv;
7072
7073         dv.major_version = DRV_VERSION_MAJOR;
7074         dv.minor_version = DRV_VERSION_MINOR;
7075         dv.build_version = DRV_VERSION_BUILD;
7076         dv.subbuild_version = 0;
7077         strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
7078         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
7079 }
7080
7081 /**
7082  * i40e_get_oem_version - get OEM specific version information
7083  * @hw: pointer to the hardware structure
7084  **/
7085 static void i40e_get_oem_version(struct i40e_hw *hw)
7086 {
7087         u16 block_offset = 0xffff;
7088         u16 block_length = 0;
7089         u16 capabilities = 0;
7090         u16 gen_snap = 0;
7091         u16 release = 0;
7092
7093 #define I40E_SR_NVM_OEM_VERSION_PTR             0x1B
7094 #define I40E_NVM_OEM_LENGTH_OFFSET              0x00
7095 #define I40E_NVM_OEM_CAPABILITIES_OFFSET        0x01
7096 #define I40E_NVM_OEM_GEN_OFFSET                 0x02
7097 #define I40E_NVM_OEM_RELEASE_OFFSET             0x03
7098 #define I40E_NVM_OEM_CAPABILITIES_MASK          0x000F
7099 #define I40E_NVM_OEM_LENGTH                     3
7100
7101         /* Check if pointer to OEM version block is valid. */
7102         i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
7103         if (block_offset == 0xffff)
7104                 return;
7105
7106         /* Check if OEM version block has correct length. */
7107         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
7108                            &block_length);
7109         if (block_length < I40E_NVM_OEM_LENGTH)
7110                 return;
7111
7112         /* Check if OEM version format is as expected. */
7113         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
7114                            &capabilities);
7115         if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
7116                 return;
7117
7118         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
7119                            &gen_snap);
7120         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
7121                            &release);
7122         hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
7123         hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
7124 }
7125
7126 /**
7127  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
7128  * @pf: board private structure
7129  **/
7130 static int i40e_reset(struct i40e_pf *pf)
7131 {
7132         struct i40e_hw *hw = &pf->hw;
7133         i40e_status ret;
7134
7135         ret = i40e_pf_reset(hw);
7136         if (ret) {
7137                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
7138                 set_bit(__I40E_RESET_FAILED, pf->state);
7139                 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
7140         } else {
7141                 pf->pfr_count++;
7142         }
7143         return ret;
7144 }
7145
7146 /**
7147  * i40e_rebuild - rebuild using a saved config
7148  * @pf: board private structure
7149  * @reinit: if the Main VSI needs to re-initialized.
7150  * @lock_acquired: indicates whether or not the lock has been acquired
7151  * before this function was called.
7152  **/
7153 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
7154 {
7155         struct i40e_hw *hw = &pf->hw;
7156         u8 set_fc_aq_fail = 0;
7157         i40e_status ret;
7158         u32 val;
7159         int v;
7160
7161         if (test_bit(__I40E_DOWN, pf->state))
7162                 goto clear_recovery;
7163         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
7164
7165         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
7166         ret = i40e_init_adminq(&pf->hw);
7167         if (ret) {
7168                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
7169                          i40e_stat_str(&pf->hw, ret),
7170                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7171                 goto clear_recovery;
7172         }
7173         i40e_get_oem_version(&pf->hw);
7174
7175         /* re-verify the eeprom if we just had an EMP reset */
7176         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
7177                 i40e_verify_eeprom(pf);
7178
7179         i40e_clear_pxe_mode(hw);
7180         ret = i40e_get_capabilities(pf);
7181         if (ret)
7182                 goto end_core_reset;
7183
7184         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
7185                                 hw->func_caps.num_rx_qp, 0, 0);
7186         if (ret) {
7187                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
7188                 goto end_core_reset;
7189         }
7190         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
7191         if (ret) {
7192                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
7193                 goto end_core_reset;
7194         }
7195
7196 #ifdef CONFIG_I40E_DCB
7197         ret = i40e_init_pf_dcb(pf);
7198         if (ret) {
7199                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret);
7200                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
7201                 /* Continue without DCB enabled */
7202         }
7203 #endif /* CONFIG_I40E_DCB */
7204         /* do basic switch setup */
7205         if (!lock_acquired)
7206                 rtnl_lock();
7207         ret = i40e_setup_pf_switch(pf, reinit);
7208         if (ret)
7209                 goto end_unlock;
7210
7211         /* The driver only wants link up/down and module qualification
7212          * reports from firmware.  Note the negative logic.
7213          */
7214         ret = i40e_aq_set_phy_int_mask(&pf->hw,
7215                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
7216                                          I40E_AQ_EVENT_MEDIA_NA |
7217                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
7218         if (ret)
7219                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
7220                          i40e_stat_str(&pf->hw, ret),
7221                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7222
7223         /* make sure our flow control settings are restored */
7224         ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true);
7225         if (ret)
7226                 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n",
7227                         i40e_stat_str(&pf->hw, ret),
7228                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7229
7230         /* Rebuild the VSIs and VEBs that existed before reset.
7231          * They are still in our local switch element arrays, so only
7232          * need to rebuild the switch model in the HW.
7233          *
7234          * If there were VEBs but the reconstitution failed, we'll try
7235          * try to recover minimal use by getting the basic PF VSI working.
7236          */
7237         if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) {
7238                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
7239                 /* find the one VEB connected to the MAC, and find orphans */
7240                 for (v = 0; v < I40E_MAX_VEB; v++) {
7241                         if (!pf->veb[v])
7242                                 continue;
7243
7244                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
7245                             pf->veb[v]->uplink_seid == 0) {
7246                                 ret = i40e_reconstitute_veb(pf->veb[v]);
7247
7248                                 if (!ret)
7249                                         continue;
7250
7251                                 /* If Main VEB failed, we're in deep doodoo,
7252                                  * so give up rebuilding the switch and set up
7253                                  * for minimal rebuild of PF VSI.
7254                                  * If orphan failed, we'll report the error
7255                                  * but try to keep going.
7256                                  */
7257                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
7258                                         dev_info(&pf->pdev->dev,
7259                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
7260                                                  ret);
7261                                         pf->vsi[pf->lan_vsi]->uplink_seid
7262                                                                 = pf->mac_seid;
7263                                         break;
7264                                 } else if (pf->veb[v]->uplink_seid == 0) {
7265                                         dev_info(&pf->pdev->dev,
7266                                                  "rebuild of orphan VEB failed: %d\n",
7267                                                  ret);
7268                                 }
7269                         }
7270                 }
7271         }
7272
7273         if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) {
7274                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
7275                 /* no VEB, so rebuild only the Main VSI */
7276                 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]);
7277                 if (ret) {
7278                         dev_info(&pf->pdev->dev,
7279                                  "rebuild of Main VSI failed: %d\n", ret);
7280                         goto end_unlock;
7281                 }
7282         }
7283
7284         /* Reconfigure hardware for allowing smaller MSS in the case
7285          * of TSO, so that we avoid the MDD being fired and causing
7286          * a reset in the case of small MSS+TSO.
7287          */
7288 #define I40E_REG_MSS          0x000E64DC
7289 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
7290 #define I40E_64BYTE_MSS       0x400000
7291         val = rd32(hw, I40E_REG_MSS);
7292         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
7293                 val &= ~I40E_REG_MSS_MIN_MASK;
7294                 val |= I40E_64BYTE_MSS;
7295                 wr32(hw, I40E_REG_MSS, val);
7296         }
7297
7298         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
7299                 msleep(75);
7300                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
7301                 if (ret)
7302                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
7303                                  i40e_stat_str(&pf->hw, ret),
7304                                  i40e_aq_str(&pf->hw,
7305                                              pf->hw.aq.asq_last_status));
7306         }
7307         /* reinit the misc interrupt */
7308         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7309                 ret = i40e_setup_misc_vector(pf);
7310
7311         /* Add a filter to drop all Flow control frames from any VSI from being
7312          * transmitted. By doing so we stop a malicious VF from sending out
7313          * PAUSE or PFC frames and potentially controlling traffic for other
7314          * PF/VF VSIs.
7315          * The FW can still send Flow control frames if enabled.
7316          */
7317         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
7318                                                        pf->main_vsi_seid);
7319
7320         /* restart the VSIs that were rebuilt and running before the reset */
7321         i40e_pf_unquiesce_all_vsi(pf);
7322
7323         /* Release the RTNL lock before we start resetting VFs */
7324         if (!lock_acquired)
7325                 rtnl_unlock();
7326
7327         i40e_reset_all_vfs(pf, true);
7328
7329         /* tell the firmware that we're starting */
7330         i40e_send_version(pf);
7331
7332         /* We've already released the lock, so don't do it again */
7333         goto end_core_reset;
7334
7335 end_unlock:
7336         if (!lock_acquired)
7337                 rtnl_unlock();
7338 end_core_reset:
7339         clear_bit(__I40E_RESET_FAILED, pf->state);
7340 clear_recovery:
7341         clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
7342 }
7343
7344 /**
7345  * i40e_reset_and_rebuild - reset and rebuild using a saved config
7346  * @pf: board private structure
7347  * @reinit: if the Main VSI needs to re-initialized.
7348  * @lock_acquired: indicates whether or not the lock has been acquired
7349  * before this function was called.
7350  **/
7351 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
7352                                    bool lock_acquired)
7353 {
7354         int ret;
7355         /* Now we wait for GRST to settle out.
7356          * We don't have to delete the VEBs or VSIs from the hw switch
7357          * because the reset will make them disappear.
7358          */
7359         ret = i40e_reset(pf);
7360         if (!ret)
7361                 i40e_rebuild(pf, reinit, lock_acquired);
7362 }
7363
7364 /**
7365  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
7366  * @pf: board private structure
7367  *
7368  * Close up the VFs and other things in prep for a Core Reset,
7369  * then get ready to rebuild the world.
7370  * @lock_acquired: indicates whether or not the lock has been acquired
7371  * before this function was called.
7372  **/
7373 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
7374 {
7375         i40e_prep_for_reset(pf, lock_acquired);
7376         i40e_reset_and_rebuild(pf, false, lock_acquired);
7377 }
7378
7379 /**
7380  * i40e_handle_mdd_event
7381  * @pf: pointer to the PF structure
7382  *
7383  * Called from the MDD irq handler to identify possibly malicious vfs
7384  **/
7385 static void i40e_handle_mdd_event(struct i40e_pf *pf)
7386 {
7387         struct i40e_hw *hw = &pf->hw;
7388         bool mdd_detected = false;
7389         bool pf_mdd_detected = false;
7390         struct i40e_vf *vf;
7391         u32 reg;
7392         int i;
7393
7394         if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
7395                 return;
7396
7397         /* find what triggered the MDD event */
7398         reg = rd32(hw, I40E_GL_MDET_TX);
7399         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
7400                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
7401                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
7402                 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
7403                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
7404                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
7405                                 I40E_GL_MDET_TX_EVENT_SHIFT;
7406                 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
7407                                 I40E_GL_MDET_TX_QUEUE_SHIFT) -
7408                                 pf->hw.func_caps.base_queue;
7409                 if (netif_msg_tx_err(pf))
7410                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
7411                                  event, queue, pf_num, vf_num);
7412                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
7413                 mdd_detected = true;
7414         }
7415         reg = rd32(hw, I40E_GL_MDET_RX);
7416         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
7417                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
7418                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
7419                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
7420                                 I40E_GL_MDET_RX_EVENT_SHIFT;
7421                 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
7422                                 I40E_GL_MDET_RX_QUEUE_SHIFT) -
7423                                 pf->hw.func_caps.base_queue;
7424                 if (netif_msg_rx_err(pf))
7425                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
7426                                  event, queue, func);
7427                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
7428                 mdd_detected = true;
7429         }
7430
7431         if (mdd_detected) {
7432                 reg = rd32(hw, I40E_PF_MDET_TX);
7433                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
7434                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
7435                         dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n");
7436                         pf_mdd_detected = true;
7437                 }
7438                 reg = rd32(hw, I40E_PF_MDET_RX);
7439                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
7440                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
7441                         dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n");
7442                         pf_mdd_detected = true;
7443                 }
7444                 /* Queue belongs to the PF, initiate a reset */
7445                 if (pf_mdd_detected) {
7446                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
7447                         i40e_service_event_schedule(pf);
7448                 }
7449         }
7450
7451         /* see if one of the VFs needs its hand slapped */
7452         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
7453                 vf = &(pf->vf[i]);
7454                 reg = rd32(hw, I40E_VP_MDET_TX(i));
7455                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
7456                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
7457                         vf->num_mdd_events++;
7458                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
7459                                  i);
7460                 }
7461
7462                 reg = rd32(hw, I40E_VP_MDET_RX(i));
7463                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
7464                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
7465                         vf->num_mdd_events++;
7466                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
7467                                  i);
7468                 }
7469
7470                 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
7471                         dev_info(&pf->pdev->dev,
7472                                  "Too many MDD events on VF %d, disabled\n", i);
7473                         dev_info(&pf->pdev->dev,
7474                                  "Use PF Control I/F to re-enable the VF\n");
7475                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
7476                 }
7477         }
7478
7479         /* re-enable mdd interrupt cause */
7480         clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
7481         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
7482         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
7483         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
7484         i40e_flush(hw);
7485 }
7486
7487 static const char *i40e_tunnel_name(struct i40e_udp_port_config *port)
7488 {
7489         switch (port->type) {
7490         case UDP_TUNNEL_TYPE_VXLAN:
7491                 return "vxlan";
7492         case UDP_TUNNEL_TYPE_GENEVE:
7493                 return "geneve";
7494         default:
7495                 return "unknown";
7496         }
7497 }
7498
7499 /**
7500  * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters
7501  * @pf: board private structure
7502  **/
7503 static void i40e_sync_udp_filters(struct i40e_pf *pf)
7504 {
7505         int i;
7506
7507         /* loop through and set pending bit for all active UDP filters */
7508         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
7509                 if (pf->udp_ports[i].port)
7510                         pf->pending_udp_bitmap |= BIT_ULL(i);
7511         }
7512
7513         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
7514 }
7515
7516 /**
7517  * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW
7518  * @pf: board private structure
7519  **/
7520 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
7521 {
7522         struct i40e_hw *hw = &pf->hw;
7523         i40e_status ret;
7524         u16 port;
7525         int i;
7526
7527         if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC))
7528                 return;
7529
7530         pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC;
7531
7532         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
7533                 if (pf->pending_udp_bitmap & BIT_ULL(i)) {
7534                         pf->pending_udp_bitmap &= ~BIT_ULL(i);
7535                         port = pf->udp_ports[i].port;
7536                         if (port)
7537                                 ret = i40e_aq_add_udp_tunnel(hw, port,
7538                                                         pf->udp_ports[i].type,
7539                                                         NULL, NULL);
7540                         else
7541                                 ret = i40e_aq_del_udp_tunnel(hw, i, NULL);
7542
7543                         if (ret) {
7544                                 dev_info(&pf->pdev->dev,
7545                                          "%s %s port %d, index %d failed, err %s aq_err %s\n",
7546                                          i40e_tunnel_name(&pf->udp_ports[i]),
7547                                          port ? "add" : "delete",
7548                                          port, i,
7549                                          i40e_stat_str(&pf->hw, ret),
7550                                          i40e_aq_str(&pf->hw,
7551                                                      pf->hw.aq.asq_last_status));
7552                                 pf->udp_ports[i].port = 0;
7553                         }
7554                 }
7555         }
7556 }
7557
7558 /**
7559  * i40e_service_task - Run the driver's async subtasks
7560  * @work: pointer to work_struct containing our data
7561  **/
7562 static void i40e_service_task(struct work_struct *work)
7563 {
7564         struct i40e_pf *pf = container_of(work,
7565                                           struct i40e_pf,
7566                                           service_task);
7567         unsigned long start_time = jiffies;
7568
7569         /* don't bother with service tasks if a reset is in progress */
7570         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
7571                 return;
7572
7573         if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
7574                 return;
7575
7576         i40e_detect_recover_hung(pf);
7577         i40e_sync_filters_subtask(pf);
7578         i40e_reset_subtask(pf);
7579         i40e_handle_mdd_event(pf);
7580         i40e_vc_process_vflr_event(pf);
7581         i40e_watchdog_subtask(pf);
7582         i40e_fdir_reinit_subtask(pf);
7583         if (pf->flags & I40E_FLAG_CLIENT_RESET) {
7584                 /* Client subtask will reopen next time through. */
7585                 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true);
7586                 pf->flags &= ~I40E_FLAG_CLIENT_RESET;
7587         } else {
7588                 i40e_client_subtask(pf);
7589                 if (pf->flags & I40E_FLAG_CLIENT_L2_CHANGE) {
7590                         i40e_notify_client_of_l2_param_changes(
7591                                                         pf->vsi[pf->lan_vsi]);
7592                         pf->flags &= ~I40E_FLAG_CLIENT_L2_CHANGE;
7593                 }
7594         }
7595         i40e_sync_filters_subtask(pf);
7596         i40e_sync_udp_filters_subtask(pf);
7597         i40e_clean_adminq_subtask(pf);
7598
7599         /* flush memory to make sure state is correct before next watchdog */
7600         smp_mb__before_atomic();
7601         clear_bit(__I40E_SERVICE_SCHED, pf->state);
7602
7603         /* If the tasks have taken longer than one timer cycle or there
7604          * is more work to be done, reschedule the service task now
7605          * rather than wait for the timer to tick again.
7606          */
7607         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
7608             test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)             ||
7609             test_bit(__I40E_MDD_EVENT_PENDING, pf->state)                ||
7610             test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
7611                 i40e_service_event_schedule(pf);
7612 }
7613
7614 /**
7615  * i40e_service_timer - timer callback
7616  * @data: pointer to PF struct
7617  **/
7618 static void i40e_service_timer(unsigned long data)
7619 {
7620         struct i40e_pf *pf = (struct i40e_pf *)data;
7621
7622         mod_timer(&pf->service_timer,
7623                   round_jiffies(jiffies + pf->service_timer_period));
7624         i40e_service_event_schedule(pf);
7625 }
7626
7627 /**
7628  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
7629  * @vsi: the VSI being configured
7630  **/
7631 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
7632 {
7633         struct i40e_pf *pf = vsi->back;
7634
7635         switch (vsi->type) {
7636         case I40E_VSI_MAIN:
7637                 vsi->alloc_queue_pairs = pf->num_lan_qps;
7638                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7639                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7640                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7641                         vsi->num_q_vectors = pf->num_lan_msix;
7642                 else
7643                         vsi->num_q_vectors = 1;
7644
7645                 break;
7646
7647         case I40E_VSI_FDIR:
7648                 vsi->alloc_queue_pairs = 1;
7649                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
7650                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7651                 vsi->num_q_vectors = pf->num_fdsb_msix;
7652                 break;
7653
7654         case I40E_VSI_VMDQ2:
7655                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
7656                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7657                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7658                 vsi->num_q_vectors = pf->num_vmdq_msix;
7659                 break;
7660
7661         case I40E_VSI_SRIOV:
7662                 vsi->alloc_queue_pairs = pf->num_vf_qps;
7663                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7664                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7665                 break;
7666
7667         default:
7668                 WARN_ON(1);
7669                 return -ENODATA;
7670         }
7671
7672         return 0;
7673 }
7674
7675 /**
7676  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
7677  * @type: VSI pointer
7678  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
7679  *
7680  * On error: returns error code (negative)
7681  * On success: returns 0
7682  **/
7683 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
7684 {
7685         struct i40e_ring **next_rings;
7686         int size;
7687         int ret = 0;
7688
7689         /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
7690         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
7691                (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
7692         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
7693         if (!vsi->tx_rings)
7694                 return -ENOMEM;
7695         next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
7696         if (i40e_enabled_xdp_vsi(vsi)) {
7697                 vsi->xdp_rings = next_rings;
7698                 next_rings += vsi->alloc_queue_pairs;
7699         }
7700         vsi->rx_rings = next_rings;
7701
7702         if (alloc_qvectors) {
7703                 /* allocate memory for q_vector pointers */
7704                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
7705                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
7706                 if (!vsi->q_vectors) {
7707                         ret = -ENOMEM;
7708                         goto err_vectors;
7709                 }
7710         }
7711         return ret;
7712
7713 err_vectors:
7714         kfree(vsi->tx_rings);
7715         return ret;
7716 }
7717
7718 /**
7719  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
7720  * @pf: board private structure
7721  * @type: type of VSI
7722  *
7723  * On error: returns error code (negative)
7724  * On success: returns vsi index in PF (positive)
7725  **/
7726 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
7727 {
7728         int ret = -ENODEV;
7729         struct i40e_vsi *vsi;
7730         int vsi_idx;
7731         int i;
7732
7733         /* Need to protect the allocation of the VSIs at the PF level */
7734         mutex_lock(&pf->switch_mutex);
7735
7736         /* VSI list may be fragmented if VSI creation/destruction has
7737          * been happening.  We can afford to do a quick scan to look
7738          * for any free VSIs in the list.
7739          *
7740          * find next empty vsi slot, looping back around if necessary
7741          */
7742         i = pf->next_vsi;
7743         while (i < pf->num_alloc_vsi && pf->vsi[i])
7744                 i++;
7745         if (i >= pf->num_alloc_vsi) {
7746                 i = 0;
7747                 while (i < pf->next_vsi && pf->vsi[i])
7748                         i++;
7749         }
7750
7751         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
7752                 vsi_idx = i;             /* Found one! */
7753         } else {
7754                 ret = -ENODEV;
7755                 goto unlock_pf;  /* out of VSI slots! */
7756         }
7757         pf->next_vsi = ++i;
7758
7759         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
7760         if (!vsi) {
7761                 ret = -ENOMEM;
7762                 goto unlock_pf;
7763         }
7764         vsi->type = type;
7765         vsi->back = pf;
7766         set_bit(__I40E_VSI_DOWN, vsi->state);
7767         vsi->flags = 0;
7768         vsi->idx = vsi_idx;
7769         vsi->int_rate_limit = 0;
7770         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
7771                                 pf->rss_table_size : 64;
7772         vsi->netdev_registered = false;
7773         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
7774         hash_init(vsi->mac_filter_hash);
7775         vsi->irqs_ready = false;
7776
7777         ret = i40e_set_num_rings_in_vsi(vsi);
7778         if (ret)
7779                 goto err_rings;
7780
7781         ret = i40e_vsi_alloc_arrays(vsi, true);
7782         if (ret)
7783                 goto err_rings;
7784
7785         /* Setup default MSIX irq handler for VSI */
7786         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
7787
7788         /* Initialize VSI lock */
7789         spin_lock_init(&vsi->mac_filter_hash_lock);
7790         pf->vsi[vsi_idx] = vsi;
7791         ret = vsi_idx;
7792         goto unlock_pf;
7793
7794 err_rings:
7795         pf->next_vsi = i - 1;
7796         kfree(vsi);
7797 unlock_pf:
7798         mutex_unlock(&pf->switch_mutex);
7799         return ret;
7800 }
7801
7802 /**
7803  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
7804  * @type: VSI pointer
7805  * @free_qvectors: a bool to specify if q_vectors need to be freed.
7806  *
7807  * On error: returns error code (negative)
7808  * On success: returns 0
7809  **/
7810 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
7811 {
7812         /* free the ring and vector containers */
7813         if (free_qvectors) {
7814                 kfree(vsi->q_vectors);
7815                 vsi->q_vectors = NULL;
7816         }
7817         kfree(vsi->tx_rings);
7818         vsi->tx_rings = NULL;
7819         vsi->rx_rings = NULL;
7820         vsi->xdp_rings = NULL;
7821 }
7822
7823 /**
7824  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
7825  * and lookup table
7826  * @vsi: Pointer to VSI structure
7827  */
7828 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
7829 {
7830         if (!vsi)
7831                 return;
7832
7833         kfree(vsi->rss_hkey_user);
7834         vsi->rss_hkey_user = NULL;
7835
7836         kfree(vsi->rss_lut_user);
7837         vsi->rss_lut_user = NULL;
7838 }
7839
7840 /**
7841  * i40e_vsi_clear - Deallocate the VSI provided
7842  * @vsi: the VSI being un-configured
7843  **/
7844 static int i40e_vsi_clear(struct i40e_vsi *vsi)
7845 {
7846         struct i40e_pf *pf;
7847
7848         if (!vsi)
7849                 return 0;
7850
7851         if (!vsi->back)
7852                 goto free_vsi;
7853         pf = vsi->back;
7854
7855         mutex_lock(&pf->switch_mutex);
7856         if (!pf->vsi[vsi->idx]) {
7857                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
7858                         vsi->idx, vsi->idx, vsi, vsi->type);
7859                 goto unlock_vsi;
7860         }
7861
7862         if (pf->vsi[vsi->idx] != vsi) {
7863                 dev_err(&pf->pdev->dev,
7864                         "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
7865                         pf->vsi[vsi->idx]->idx,
7866                         pf->vsi[vsi->idx],
7867                         pf->vsi[vsi->idx]->type,
7868                         vsi->idx, vsi, vsi->type);
7869                 goto unlock_vsi;
7870         }
7871
7872         /* updates the PF for this cleared vsi */
7873         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
7874         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
7875
7876         i40e_vsi_free_arrays(vsi, true);
7877         i40e_clear_rss_config_user(vsi);
7878
7879         pf->vsi[vsi->idx] = NULL;
7880         if (vsi->idx < pf->next_vsi)
7881                 pf->next_vsi = vsi->idx;
7882
7883 unlock_vsi:
7884         mutex_unlock(&pf->switch_mutex);
7885 free_vsi:
7886         kfree(vsi);
7887
7888         return 0;
7889 }
7890
7891 /**
7892  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
7893  * @vsi: the VSI being cleaned
7894  **/
7895 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
7896 {
7897         int i;
7898
7899         if (vsi->tx_rings && vsi->tx_rings[0]) {
7900                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7901                         kfree_rcu(vsi->tx_rings[i], rcu);
7902                         vsi->tx_rings[i] = NULL;
7903                         vsi->rx_rings[i] = NULL;
7904                         if (vsi->xdp_rings)
7905                                 vsi->xdp_rings[i] = NULL;
7906                 }
7907         }
7908 }
7909
7910 /**
7911  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
7912  * @vsi: the VSI being configured
7913  **/
7914 static int i40e_alloc_rings(struct i40e_vsi *vsi)
7915 {
7916         int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
7917         struct i40e_pf *pf = vsi->back;
7918         struct i40e_ring *ring;
7919
7920         /* Set basic values in the rings to be used later during open() */
7921         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7922                 /* allocate space for both Tx and Rx in one shot */
7923                 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
7924                 if (!ring)
7925                         goto err_out;
7926
7927                 ring->queue_index = i;
7928                 ring->reg_idx = vsi->base_queue + i;
7929                 ring->ring_active = false;
7930                 ring->vsi = vsi;
7931                 ring->netdev = vsi->netdev;
7932                 ring->dev = &pf->pdev->dev;
7933                 ring->count = vsi->num_desc;
7934                 ring->size = 0;
7935                 ring->dcb_tc = 0;
7936                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
7937                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
7938                 ring->tx_itr_setting = pf->tx_itr_default;
7939                 vsi->tx_rings[i] = ring++;
7940
7941                 if (!i40e_enabled_xdp_vsi(vsi))
7942                         goto setup_rx;
7943
7944                 ring->queue_index = vsi->alloc_queue_pairs + i;
7945                 ring->reg_idx = vsi->base_queue + ring->queue_index;
7946                 ring->ring_active = false;
7947                 ring->vsi = vsi;
7948                 ring->netdev = NULL;
7949                 ring->dev = &pf->pdev->dev;
7950                 ring->count = vsi->num_desc;
7951                 ring->size = 0;
7952                 ring->dcb_tc = 0;
7953                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
7954                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
7955                 set_ring_xdp(ring);
7956                 ring->tx_itr_setting = pf->tx_itr_default;
7957                 vsi->xdp_rings[i] = ring++;
7958
7959 setup_rx:
7960                 ring->queue_index = i;
7961                 ring->reg_idx = vsi->base_queue + i;
7962                 ring->ring_active = false;
7963                 ring->vsi = vsi;
7964                 ring->netdev = vsi->netdev;
7965                 ring->dev = &pf->pdev->dev;
7966                 ring->count = vsi->num_desc;
7967                 ring->size = 0;
7968                 ring->dcb_tc = 0;
7969                 ring->rx_itr_setting = pf->rx_itr_default;
7970                 vsi->rx_rings[i] = ring;
7971         }
7972
7973         return 0;
7974
7975 err_out:
7976         i40e_vsi_clear_rings(vsi);
7977         return -ENOMEM;
7978 }
7979
7980 /**
7981  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
7982  * @pf: board private structure
7983  * @vectors: the number of MSI-X vectors to request
7984  *
7985  * Returns the number of vectors reserved, or error
7986  **/
7987 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
7988 {
7989         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
7990                                         I40E_MIN_MSIX, vectors);
7991         if (vectors < 0) {
7992                 dev_info(&pf->pdev->dev,
7993                          "MSI-X vector reservation failed: %d\n", vectors);
7994                 vectors = 0;
7995         }
7996
7997         return vectors;
7998 }
7999
8000 /**
8001  * i40e_init_msix - Setup the MSIX capability
8002  * @pf: board private structure
8003  *
8004  * Work with the OS to set up the MSIX vectors needed.
8005  *
8006  * Returns the number of vectors reserved or negative on failure
8007  **/
8008 static int i40e_init_msix(struct i40e_pf *pf)
8009 {
8010         struct i40e_hw *hw = &pf->hw;
8011         int cpus, extra_vectors;
8012         int vectors_left;
8013         int v_budget, i;
8014         int v_actual;
8015         int iwarp_requested = 0;
8016
8017         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
8018                 return -ENODEV;
8019
8020         /* The number of vectors we'll request will be comprised of:
8021          *   - Add 1 for "other" cause for Admin Queue events, etc.
8022          *   - The number of LAN queue pairs
8023          *      - Queues being used for RSS.
8024          *              We don't need as many as max_rss_size vectors.
8025          *              use rss_size instead in the calculation since that
8026          *              is governed by number of cpus in the system.
8027          *      - assumes symmetric Tx/Rx pairing
8028          *   - The number of VMDq pairs
8029          *   - The CPU count within the NUMA node if iWARP is enabled
8030          * Once we count this up, try the request.
8031          *
8032          * If we can't get what we want, we'll simplify to nearly nothing
8033          * and try again.  If that still fails, we punt.
8034          */
8035         vectors_left = hw->func_caps.num_msix_vectors;
8036         v_budget = 0;
8037
8038         /* reserve one vector for miscellaneous handler */
8039         if (vectors_left) {
8040                 v_budget++;
8041                 vectors_left--;
8042         }
8043
8044         /* reserve some vectors for the main PF traffic queues. Initially we
8045          * only reserve at most 50% of the available vectors, in the case that
8046          * the number of online CPUs is large. This ensures that we can enable
8047          * extra features as well. Once we've enabled the other features, we
8048          * will use any remaining vectors to reach as close as we can to the
8049          * number of online CPUs.
8050          */
8051         cpus = num_online_cpus();
8052         pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
8053         vectors_left -= pf->num_lan_msix;
8054
8055         /* reserve one vector for sideband flow director */
8056         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8057                 if (vectors_left) {
8058                         pf->num_fdsb_msix = 1;
8059                         v_budget++;
8060                         vectors_left--;
8061                 } else {
8062                         pf->num_fdsb_msix = 0;
8063                 }
8064         }
8065
8066         /* can we reserve enough for iWARP? */
8067         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
8068                 iwarp_requested = pf->num_iwarp_msix;
8069
8070                 if (!vectors_left)
8071                         pf->num_iwarp_msix = 0;
8072                 else if (vectors_left < pf->num_iwarp_msix)
8073                         pf->num_iwarp_msix = 1;
8074                 v_budget += pf->num_iwarp_msix;
8075                 vectors_left -= pf->num_iwarp_msix;
8076         }
8077
8078         /* any vectors left over go for VMDq support */
8079         if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
8080                 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps;
8081                 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted);
8082
8083                 if (!vectors_left) {
8084                         pf->num_vmdq_msix = 0;
8085                         pf->num_vmdq_qps = 0;
8086                 } else {
8087                         /* if we're short on vectors for what's desired, we limit
8088                          * the queues per vmdq.  If this is still more than are
8089                          * available, the user will need to change the number of
8090                          * queues/vectors used by the PF later with the ethtool
8091                          * channels command
8092                          */
8093                         if (vmdq_vecs < vmdq_vecs_wanted)
8094                                 pf->num_vmdq_qps = 1;
8095                         pf->num_vmdq_msix = pf->num_vmdq_qps;
8096
8097                         v_budget += vmdq_vecs;
8098                         vectors_left -= vmdq_vecs;
8099                 }
8100         }
8101
8102         /* On systems with a large number of SMP cores, we previously limited
8103          * the number of vectors for num_lan_msix to be at most 50% of the
8104          * available vectors, to allow for other features. Now, we add back
8105          * the remaining vectors. However, we ensure that the total
8106          * num_lan_msix will not exceed num_online_cpus(). To do this, we
8107          * calculate the number of vectors we can add without going over the
8108          * cap of CPUs. For systems with a small number of CPUs this will be
8109          * zero.
8110          */
8111         extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
8112         pf->num_lan_msix += extra_vectors;
8113         vectors_left -= extra_vectors;
8114
8115         WARN(vectors_left < 0,
8116              "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
8117
8118         v_budget += pf->num_lan_msix;
8119         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
8120                                    GFP_KERNEL);
8121         if (!pf->msix_entries)
8122                 return -ENOMEM;
8123
8124         for (i = 0; i < v_budget; i++)
8125                 pf->msix_entries[i].entry = i;
8126         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
8127
8128         if (v_actual < I40E_MIN_MSIX) {
8129                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
8130                 kfree(pf->msix_entries);
8131                 pf->msix_entries = NULL;
8132                 pci_disable_msix(pf->pdev);
8133                 return -ENODEV;
8134
8135         } else if (v_actual == I40E_MIN_MSIX) {
8136                 /* Adjust for minimal MSIX use */
8137                 pf->num_vmdq_vsis = 0;
8138                 pf->num_vmdq_qps = 0;
8139                 pf->num_lan_qps = 1;
8140                 pf->num_lan_msix = 1;
8141
8142         } else if (!vectors_left) {
8143                 /* If we have limited resources, we will start with no vectors
8144                  * for the special features and then allocate vectors to some
8145                  * of these features based on the policy and at the end disable
8146                  * the features that did not get any vectors.
8147                  */
8148                 int vec;
8149
8150                 dev_info(&pf->pdev->dev,
8151                          "MSI-X vector limit reached, attempting to redistribute vectors\n");
8152                 /* reserve the misc vector */
8153                 vec = v_actual - 1;
8154
8155                 /* Scale vector usage down */
8156                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
8157                 pf->num_vmdq_vsis = 1;
8158                 pf->num_vmdq_qps = 1;
8159
8160                 /* partition out the remaining vectors */
8161                 switch (vec) {
8162                 case 2:
8163                         pf->num_lan_msix = 1;
8164                         break;
8165                 case 3:
8166                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
8167                                 pf->num_lan_msix = 1;
8168                                 pf->num_iwarp_msix = 1;
8169                         } else {
8170                                 pf->num_lan_msix = 2;
8171                         }
8172                         break;
8173                 default:
8174                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
8175                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
8176                                                  iwarp_requested);
8177                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
8178                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
8179                         } else {
8180                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
8181                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
8182                         }
8183                         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8184                                 pf->num_fdsb_msix = 1;
8185                                 vec--;
8186                         }
8187                         pf->num_lan_msix = min_t(int,
8188                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
8189                                                               pf->num_lan_msix);
8190                         pf->num_lan_qps = pf->num_lan_msix;
8191                         break;
8192                 }
8193         }
8194
8195         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
8196             (pf->num_fdsb_msix == 0)) {
8197                 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
8198                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8199         }
8200         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
8201             (pf->num_vmdq_msix == 0)) {
8202                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
8203                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
8204         }
8205
8206         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
8207             (pf->num_iwarp_msix == 0)) {
8208                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
8209                 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
8210         }
8211         i40e_debug(&pf->hw, I40E_DEBUG_INIT,
8212                    "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
8213                    pf->num_lan_msix,
8214                    pf->num_vmdq_msix * pf->num_vmdq_vsis,
8215                    pf->num_fdsb_msix,
8216                    pf->num_iwarp_msix);
8217
8218         return v_actual;
8219 }
8220
8221 /**
8222  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
8223  * @vsi: the VSI being configured
8224  * @v_idx: index of the vector in the vsi struct
8225  * @cpu: cpu to be used on affinity_mask
8226  *
8227  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
8228  **/
8229 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu)
8230 {
8231         struct i40e_q_vector *q_vector;
8232
8233         /* allocate q_vector */
8234         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
8235         if (!q_vector)
8236                 return -ENOMEM;
8237
8238         q_vector->vsi = vsi;
8239         q_vector->v_idx = v_idx;
8240         cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
8241
8242         if (vsi->netdev)
8243                 netif_napi_add(vsi->netdev, &q_vector->napi,
8244                                i40e_napi_poll, NAPI_POLL_WEIGHT);
8245
8246         q_vector->rx.latency_range = I40E_LOW_LATENCY;
8247         q_vector->tx.latency_range = I40E_LOW_LATENCY;
8248
8249         /* tie q_vector and vsi together */
8250         vsi->q_vectors[v_idx] = q_vector;
8251
8252         return 0;
8253 }
8254
8255 /**
8256  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
8257  * @vsi: the VSI being configured
8258  *
8259  * We allocate one q_vector per queue interrupt.  If allocation fails we
8260  * return -ENOMEM.
8261  **/
8262 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
8263 {
8264         struct i40e_pf *pf = vsi->back;
8265         int err, v_idx, num_q_vectors, current_cpu;
8266
8267         /* if not MSIX, give the one vector only to the LAN VSI */
8268         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
8269                 num_q_vectors = vsi->num_q_vectors;
8270         else if (vsi == pf->vsi[pf->lan_vsi])
8271                 num_q_vectors = 1;
8272         else
8273                 return -EINVAL;
8274
8275         current_cpu = cpumask_first(cpu_online_mask);
8276
8277         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
8278                 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu);
8279                 if (err)
8280                         goto err_out;
8281                 current_cpu = cpumask_next(current_cpu, cpu_online_mask);
8282                 if (unlikely(current_cpu >= nr_cpu_ids))
8283                         current_cpu = cpumask_first(cpu_online_mask);
8284         }
8285
8286         return 0;
8287
8288 err_out:
8289         while (v_idx--)
8290                 i40e_free_q_vector(vsi, v_idx);
8291
8292         return err;
8293 }
8294
8295 /**
8296  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
8297  * @pf: board private structure to initialize
8298  **/
8299 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
8300 {
8301         int vectors = 0;
8302         ssize_t size;
8303
8304         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
8305                 vectors = i40e_init_msix(pf);
8306                 if (vectors < 0) {
8307                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
8308                                        I40E_FLAG_IWARP_ENABLED  |
8309                                        I40E_FLAG_RSS_ENABLED    |
8310                                        I40E_FLAG_DCB_CAPABLE    |
8311                                        I40E_FLAG_DCB_ENABLED    |
8312                                        I40E_FLAG_SRIOV_ENABLED  |
8313                                        I40E_FLAG_FD_SB_ENABLED  |
8314                                        I40E_FLAG_FD_ATR_ENABLED |
8315                                        I40E_FLAG_VMDQ_ENABLED);
8316
8317                         /* rework the queue expectations without MSIX */
8318                         i40e_determine_queue_usage(pf);
8319                 }
8320         }
8321
8322         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
8323             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
8324                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
8325                 vectors = pci_enable_msi(pf->pdev);
8326                 if (vectors < 0) {
8327                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
8328                                  vectors);
8329                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
8330                 }
8331                 vectors = 1;  /* one MSI or Legacy vector */
8332         }
8333
8334         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
8335                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
8336
8337         /* set up vector assignment tracking */
8338         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
8339         pf->irq_pile = kzalloc(size, GFP_KERNEL);
8340         if (!pf->irq_pile) {
8341                 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n");
8342                 return -ENOMEM;
8343         }
8344         pf->irq_pile->num_entries = vectors;
8345         pf->irq_pile->search_hint = 0;
8346
8347         /* track first vector for misc interrupts, ignore return */
8348         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
8349
8350         return 0;
8351 }
8352
8353 /**
8354  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
8355  * @pf: board private structure
8356  *
8357  * This sets up the handler for MSIX 0, which is used to manage the
8358  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
8359  * when in MSI or Legacy interrupt mode.
8360  **/
8361 static int i40e_setup_misc_vector(struct i40e_pf *pf)
8362 {
8363         struct i40e_hw *hw = &pf->hw;
8364         int err = 0;
8365
8366         /* Only request the irq if this is the first time through, and
8367          * not when we're rebuilding after a Reset
8368          */
8369         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {
8370                 err = request_irq(pf->msix_entries[0].vector,
8371                                   i40e_intr, 0, pf->int_name, pf);
8372                 if (err) {
8373                         dev_info(&pf->pdev->dev,
8374                                  "request_irq for %s failed: %d\n",
8375                                  pf->int_name, err);
8376                         return -EFAULT;
8377                 }
8378         }
8379
8380         i40e_enable_misc_int_causes(pf);
8381
8382         /* associate no queues to the misc vector */
8383         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
8384         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
8385
8386         i40e_flush(hw);
8387
8388         i40e_irq_dynamic_enable_icr0(pf, true);
8389
8390         return err;
8391 }
8392
8393 /**
8394  * i40e_config_rss_aq - Prepare for RSS using AQ commands
8395  * @vsi: vsi structure
8396  * @seed: RSS hash seed
8397  **/
8398 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
8399                               u8 *lut, u16 lut_size)
8400 {
8401         struct i40e_pf *pf = vsi->back;
8402         struct i40e_hw *hw = &pf->hw;
8403         int ret = 0;
8404
8405         if (seed) {
8406                 struct i40e_aqc_get_set_rss_key_data *seed_dw =
8407                         (struct i40e_aqc_get_set_rss_key_data *)seed;
8408                 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
8409                 if (ret) {
8410                         dev_info(&pf->pdev->dev,
8411                                  "Cannot set RSS key, err %s aq_err %s\n",
8412                                  i40e_stat_str(hw, ret),
8413                                  i40e_aq_str(hw, hw->aq.asq_last_status));
8414                         return ret;
8415                 }
8416         }
8417         if (lut) {
8418                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
8419
8420                 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
8421                 if (ret) {
8422                         dev_info(&pf->pdev->dev,
8423                                  "Cannot set RSS lut, err %s aq_err %s\n",
8424                                  i40e_stat_str(hw, ret),
8425                                  i40e_aq_str(hw, hw->aq.asq_last_status));
8426                         return ret;
8427                 }
8428         }
8429         return ret;
8430 }
8431
8432 /**
8433  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
8434  * @vsi: Pointer to vsi structure
8435  * @seed: Buffter to store the hash keys
8436  * @lut: Buffer to store the lookup table entries
8437  * @lut_size: Size of buffer to store the lookup table entries
8438  *
8439  * Return 0 on success, negative on failure
8440  */
8441 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
8442                            u8 *lut, u16 lut_size)
8443 {
8444         struct i40e_pf *pf = vsi->back;
8445         struct i40e_hw *hw = &pf->hw;
8446         int ret = 0;
8447
8448         if (seed) {
8449                 ret = i40e_aq_get_rss_key(hw, vsi->id,
8450                         (struct i40e_aqc_get_set_rss_key_data *)seed);
8451                 if (ret) {
8452                         dev_info(&pf->pdev->dev,
8453                                  "Cannot get RSS key, err %s aq_err %s\n",
8454                                  i40e_stat_str(&pf->hw, ret),
8455                                  i40e_aq_str(&pf->hw,
8456                                              pf->hw.aq.asq_last_status));
8457                         return ret;
8458                 }
8459         }
8460
8461         if (lut) {
8462                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
8463
8464                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
8465                 if (ret) {
8466                         dev_info(&pf->pdev->dev,
8467                                  "Cannot get RSS lut, err %s aq_err %s\n",
8468                                  i40e_stat_str(&pf->hw, ret),
8469                                  i40e_aq_str(&pf->hw,
8470                                              pf->hw.aq.asq_last_status));
8471                         return ret;
8472                 }
8473         }
8474
8475         return ret;
8476 }
8477
8478 /**
8479  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
8480  * @vsi: VSI structure
8481  **/
8482 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
8483 {
8484         u8 seed[I40E_HKEY_ARRAY_SIZE];
8485         struct i40e_pf *pf = vsi->back;
8486         u8 *lut;
8487         int ret;
8488
8489         if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
8490                 return 0;
8491
8492         if (!vsi->rss_size)
8493                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
8494                                       vsi->num_queue_pairs);
8495         if (!vsi->rss_size)
8496                 return -EINVAL;
8497
8498         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8499         if (!lut)
8500                 return -ENOMEM;
8501         /* Use the user configured hash keys and lookup table if there is one,
8502          * otherwise use default
8503          */
8504         if (vsi->rss_lut_user)
8505                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
8506         else
8507                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8508         if (vsi->rss_hkey_user)
8509                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
8510         else
8511                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8512         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
8513         kfree(lut);
8514
8515         return ret;
8516 }
8517
8518 /**
8519  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
8520  * @vsi: Pointer to vsi structure
8521  * @seed: RSS hash seed
8522  * @lut: Lookup table
8523  * @lut_size: Lookup table size
8524  *
8525  * Returns 0 on success, negative on failure
8526  **/
8527 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
8528                                const u8 *lut, u16 lut_size)
8529 {
8530         struct i40e_pf *pf = vsi->back;
8531         struct i40e_hw *hw = &pf->hw;
8532         u16 vf_id = vsi->vf_id;
8533         u8 i;
8534
8535         /* Fill out hash function seed */
8536         if (seed) {
8537                 u32 *seed_dw = (u32 *)seed;
8538
8539                 if (vsi->type == I40E_VSI_MAIN) {
8540                         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8541                                 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
8542                 } else if (vsi->type == I40E_VSI_SRIOV) {
8543                         for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
8544                                 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
8545                 } else {
8546                         dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
8547                 }
8548         }
8549
8550         if (lut) {
8551                 u32 *lut_dw = (u32 *)lut;
8552
8553                 if (vsi->type == I40E_VSI_MAIN) {
8554                         if (lut_size != I40E_HLUT_ARRAY_SIZE)
8555                                 return -EINVAL;
8556                         for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8557                                 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
8558                 } else if (vsi->type == I40E_VSI_SRIOV) {
8559                         if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
8560                                 return -EINVAL;
8561                         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
8562                                 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
8563                 } else {
8564                         dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
8565                 }
8566         }
8567         i40e_flush(hw);
8568
8569         return 0;
8570 }
8571
8572 /**
8573  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
8574  * @vsi: Pointer to VSI structure
8575  * @seed: Buffer to store the keys
8576  * @lut: Buffer to store the lookup table entries
8577  * @lut_size: Size of buffer to store the lookup table entries
8578  *
8579  * Returns 0 on success, negative on failure
8580  */
8581 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
8582                             u8 *lut, u16 lut_size)
8583 {
8584         struct i40e_pf *pf = vsi->back;
8585         struct i40e_hw *hw = &pf->hw;
8586         u16 i;
8587
8588         if (seed) {
8589                 u32 *seed_dw = (u32 *)seed;
8590
8591                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8592                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
8593         }
8594         if (lut) {
8595                 u32 *lut_dw = (u32 *)lut;
8596
8597                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
8598                         return -EINVAL;
8599                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8600                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
8601         }
8602
8603         return 0;
8604 }
8605
8606 /**
8607  * i40e_config_rss - Configure RSS keys and lut
8608  * @vsi: Pointer to VSI structure
8609  * @seed: RSS hash seed
8610  * @lut: Lookup table
8611  * @lut_size: Lookup table size
8612  *
8613  * Returns 0 on success, negative on failure
8614  */
8615 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8616 {
8617         struct i40e_pf *pf = vsi->back;
8618
8619         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
8620                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
8621         else
8622                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
8623 }
8624
8625 /**
8626  * i40e_get_rss - Get RSS keys and lut
8627  * @vsi: Pointer to VSI structure
8628  * @seed: Buffer to store the keys
8629  * @lut: Buffer to store the lookup table entries
8630  * lut_size: Size of buffer to store the lookup table entries
8631  *
8632  * Returns 0 on success, negative on failure
8633  */
8634 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8635 {
8636         struct i40e_pf *pf = vsi->back;
8637
8638         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
8639                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
8640         else
8641                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
8642 }
8643
8644 /**
8645  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
8646  * @pf: Pointer to board private structure
8647  * @lut: Lookup table
8648  * @rss_table_size: Lookup table size
8649  * @rss_size: Range of queue number for hashing
8650  */
8651 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
8652                        u16 rss_table_size, u16 rss_size)
8653 {
8654         u16 i;
8655
8656         for (i = 0; i < rss_table_size; i++)
8657                 lut[i] = i % rss_size;
8658 }
8659
8660 /**
8661  * i40e_pf_config_rss - Prepare for RSS if used
8662  * @pf: board private structure
8663  **/
8664 static int i40e_pf_config_rss(struct i40e_pf *pf)
8665 {
8666         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8667         u8 seed[I40E_HKEY_ARRAY_SIZE];
8668         u8 *lut;
8669         struct i40e_hw *hw = &pf->hw;
8670         u32 reg_val;
8671         u64 hena;
8672         int ret;
8673
8674         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
8675         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
8676                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
8677         hena |= i40e_pf_get_default_rss_hena(pf);
8678
8679         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
8680         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
8681
8682         /* Determine the RSS table size based on the hardware capabilities */
8683         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
8684         reg_val = (pf->rss_table_size == 512) ?
8685                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
8686                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
8687         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
8688
8689         /* Determine the RSS size of the VSI */
8690         if (!vsi->rss_size) {
8691                 u16 qcount;
8692
8693                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
8694                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
8695         }
8696         if (!vsi->rss_size)
8697                 return -EINVAL;
8698
8699         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8700         if (!lut)
8701                 return -ENOMEM;
8702
8703         /* Use user configured lut if there is one, otherwise use default */
8704         if (vsi->rss_lut_user)
8705                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
8706         else
8707                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8708
8709         /* Use user configured hash key if there is one, otherwise
8710          * use default.
8711          */
8712         if (vsi->rss_hkey_user)
8713                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
8714         else
8715                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8716         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
8717         kfree(lut);
8718
8719         return ret;
8720 }
8721
8722 /**
8723  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
8724  * @pf: board private structure
8725  * @queue_count: the requested queue count for rss.
8726  *
8727  * returns 0 if rss is not enabled, if enabled returns the final rss queue
8728  * count which may be different from the requested queue count.
8729  * Note: expects to be called while under rtnl_lock()
8730  **/
8731 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
8732 {
8733         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8734         int new_rss_size;
8735
8736         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
8737                 return 0;
8738
8739         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
8740
8741         if (queue_count != vsi->num_queue_pairs) {
8742                 u16 qcount;
8743
8744                 vsi->req_queue_pairs = queue_count;
8745                 i40e_prep_for_reset(pf, true);
8746
8747                 pf->alloc_rss_size = new_rss_size;
8748
8749                 i40e_reset_and_rebuild(pf, true, true);
8750
8751                 /* Discard the user configured hash keys and lut, if less
8752                  * queues are enabled.
8753                  */
8754                 if (queue_count < vsi->rss_size) {
8755                         i40e_clear_rss_config_user(vsi);
8756                         dev_dbg(&pf->pdev->dev,
8757                                 "discard user configured hash keys and lut\n");
8758                 }
8759
8760                 /* Reset vsi->rss_size, as number of enabled queues changed */
8761                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
8762                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
8763
8764                 i40e_pf_config_rss(pf);
8765         }
8766         dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
8767                  vsi->req_queue_pairs, pf->rss_size_max);
8768         return pf->alloc_rss_size;
8769 }
8770
8771 /**
8772  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
8773  * @pf: board private structure
8774  **/
8775 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf)
8776 {
8777         i40e_status status;
8778         bool min_valid, max_valid;
8779         u32 max_bw, min_bw;
8780
8781         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
8782                                            &min_valid, &max_valid);
8783
8784         if (!status) {
8785                 if (min_valid)
8786                         pf->min_bw = min_bw;
8787                 if (max_valid)
8788                         pf->max_bw = max_bw;
8789         }
8790
8791         return status;
8792 }
8793
8794 /**
8795  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
8796  * @pf: board private structure
8797  **/
8798 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf)
8799 {
8800         struct i40e_aqc_configure_partition_bw_data bw_data;
8801         i40e_status status;
8802
8803         /* Set the valid bit for this PF */
8804         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
8805         bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
8806         bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
8807
8808         /* Set the new bandwidths */
8809         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
8810
8811         return status;
8812 }
8813
8814 /**
8815  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
8816  * @pf: board private structure
8817  **/
8818 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
8819 {
8820         /* Commit temporary BW setting to permanent NVM image */
8821         enum i40e_admin_queue_err last_aq_status;
8822         i40e_status ret;
8823         u16 nvm_word;
8824
8825         if (pf->hw.partition_id != 1) {
8826                 dev_info(&pf->pdev->dev,
8827                          "Commit BW only works on partition 1! This is partition %d",
8828                          pf->hw.partition_id);
8829                 ret = I40E_NOT_SUPPORTED;
8830                 goto bw_commit_out;
8831         }
8832
8833         /* Acquire NVM for read access */
8834         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
8835         last_aq_status = pf->hw.aq.asq_last_status;
8836         if (ret) {
8837                 dev_info(&pf->pdev->dev,
8838                          "Cannot acquire NVM for read access, err %s aq_err %s\n",
8839                          i40e_stat_str(&pf->hw, ret),
8840                          i40e_aq_str(&pf->hw, last_aq_status));
8841                 goto bw_commit_out;
8842         }
8843
8844         /* Read word 0x10 of NVM - SW compatibility word 1 */
8845         ret = i40e_aq_read_nvm(&pf->hw,
8846                                I40E_SR_NVM_CONTROL_WORD,
8847                                0x10, sizeof(nvm_word), &nvm_word,
8848                                false, NULL);
8849         /* Save off last admin queue command status before releasing
8850          * the NVM
8851          */
8852         last_aq_status = pf->hw.aq.asq_last_status;
8853         i40e_release_nvm(&pf->hw);
8854         if (ret) {
8855                 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
8856                          i40e_stat_str(&pf->hw, ret),
8857                          i40e_aq_str(&pf->hw, last_aq_status));
8858                 goto bw_commit_out;
8859         }
8860
8861         /* Wait a bit for NVM release to complete */
8862         msleep(50);
8863
8864         /* Acquire NVM for write access */
8865         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
8866         last_aq_status = pf->hw.aq.asq_last_status;
8867         if (ret) {
8868                 dev_info(&pf->pdev->dev,
8869                          "Cannot acquire NVM for write access, err %s aq_err %s\n",
8870                          i40e_stat_str(&pf->hw, ret),
8871                          i40e_aq_str(&pf->hw, last_aq_status));
8872                 goto bw_commit_out;
8873         }
8874         /* Write it back out unchanged to initiate update NVM,
8875          * which will force a write of the shadow (alt) RAM to
8876          * the NVM - thus storing the bandwidth values permanently.
8877          */
8878         ret = i40e_aq_update_nvm(&pf->hw,
8879                                  I40E_SR_NVM_CONTROL_WORD,
8880                                  0x10, sizeof(nvm_word),
8881                                  &nvm_word, true, NULL);
8882         /* Save off last admin queue command status before releasing
8883          * the NVM
8884          */
8885         last_aq_status = pf->hw.aq.asq_last_status;
8886         i40e_release_nvm(&pf->hw);
8887         if (ret)
8888                 dev_info(&pf->pdev->dev,
8889                          "BW settings NOT SAVED, err %s aq_err %s\n",
8890                          i40e_stat_str(&pf->hw, ret),
8891                          i40e_aq_str(&pf->hw, last_aq_status));
8892 bw_commit_out:
8893
8894         return ret;
8895 }
8896
8897 /**
8898  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
8899  * @pf: board private structure to initialize
8900  *
8901  * i40e_sw_init initializes the Adapter private data structure.
8902  * Fields are initialized based on PCI device information and
8903  * OS network device settings (MTU size).
8904  **/
8905 static int i40e_sw_init(struct i40e_pf *pf)
8906 {
8907         int err = 0;
8908         int size;
8909
8910         /* Set default capability flags */
8911         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
8912                     I40E_FLAG_MSI_ENABLED     |
8913                     I40E_FLAG_MSIX_ENABLED;
8914
8915         /* Set default ITR */
8916         pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF;
8917         pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF;
8918
8919         /* Depending on PF configurations, it is possible that the RSS
8920          * maximum might end up larger than the available queues
8921          */
8922         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
8923         pf->alloc_rss_size = 1;
8924         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
8925         pf->rss_size_max = min_t(int, pf->rss_size_max,
8926                                  pf->hw.func_caps.num_tx_qp);
8927         if (pf->hw.func_caps.rss) {
8928                 pf->flags |= I40E_FLAG_RSS_ENABLED;
8929                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
8930                                            num_online_cpus());
8931         }
8932
8933         /* MFP mode enabled */
8934         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
8935                 pf->flags |= I40E_FLAG_MFP_ENABLED;
8936                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
8937                 if (i40e_get_partition_bw_setting(pf)) {
8938                         dev_warn(&pf->pdev->dev,
8939                                  "Could not get partition bw settings\n");
8940                 } else {
8941                         dev_info(&pf->pdev->dev,
8942                                  "Partition BW Min = %8.8x, Max = %8.8x\n",
8943                                  pf->min_bw, pf->max_bw);
8944
8945                         /* nudge the Tx scheduler */
8946                         i40e_set_partition_bw_setting(pf);
8947                 }
8948         }
8949
8950         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
8951             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
8952                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
8953                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
8954                 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
8955                     pf->hw.num_partitions > 1)
8956                         dev_info(&pf->pdev->dev,
8957                                  "Flow Director Sideband mode Disabled in MFP mode\n");
8958                 else
8959                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8960                 pf->fdir_pf_filter_count =
8961                                  pf->hw.func_caps.fd_filters_guaranteed;
8962                 pf->hw.fdir_shared_filter_count =
8963                                  pf->hw.func_caps.fd_filters_best_effort;
8964         }
8965
8966         if (pf->hw.mac.type == I40E_MAC_X722) {
8967                 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
8968                                     I40E_HW_128_QP_RSS_CAPABLE |
8969                                     I40E_HW_ATR_EVICT_CAPABLE |
8970                                     I40E_HW_WB_ON_ITR_CAPABLE |
8971                                     I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
8972                                     I40E_HW_NO_PCI_LINK_CHECK |
8973                                     I40E_HW_USE_SET_LLDP_MIB |
8974                                     I40E_HW_GENEVE_OFFLOAD_CAPABLE |
8975                                     I40E_HW_PTP_L4_CAPABLE |
8976                                     I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
8977                                     I40E_HW_OUTER_UDP_CSUM_CAPABLE);
8978
8979 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
8980                 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
8981                     I40E_FDEVICT_PCTYPE_DEFAULT) {
8982                         dev_warn(&pf->pdev->dev,
8983                                  "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
8984                         pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
8985                 }
8986         } else if ((pf->hw.aq.api_maj_ver > 1) ||
8987                    ((pf->hw.aq.api_maj_ver == 1) &&
8988                     (pf->hw.aq.api_min_ver > 4))) {
8989                 /* Supported in FW API version higher than 1.4 */
8990                 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
8991         }
8992
8993         /* Enable HW ATR eviction if possible */
8994         if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
8995                 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
8996
8997         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
8998             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
8999             (pf->hw.aq.fw_maj_ver < 4))) {
9000                 pf->hw_features |= I40E_HW_RESTART_AUTONEG;
9001                 /* No DCB support  for FW < v4.33 */
9002                 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
9003         }
9004
9005         /* Disable FW LLDP if FW < v4.3 */
9006         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
9007             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
9008             (pf->hw.aq.fw_maj_ver < 4)))
9009                 pf->hw_features |= I40E_HW_STOP_FW_LLDP;
9010
9011         /* Use the FW Set LLDP MIB API if FW > v4.40 */
9012         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
9013             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
9014             (pf->hw.aq.fw_maj_ver >= 5)))
9015                 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
9016
9017         if (pf->hw.func_caps.vmdq) {
9018                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
9019                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
9020                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
9021         }
9022
9023         if (pf->hw.func_caps.iwarp) {
9024                 pf->flags |= I40E_FLAG_IWARP_ENABLED;
9025                 /* IWARP needs one extra vector for CQP just like MISC.*/
9026                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
9027         }
9028
9029 #ifdef CONFIG_PCI_IOV
9030         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
9031                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
9032                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
9033                 pf->num_req_vfs = min_t(int,
9034                                         pf->hw.func_caps.num_vfs,
9035                                         I40E_MAX_VF_COUNT);
9036         }
9037 #endif /* CONFIG_PCI_IOV */
9038         pf->eeprom_version = 0xDEAD;
9039         pf->lan_veb = I40E_NO_VEB;
9040         pf->lan_vsi = I40E_NO_VSI;
9041
9042         /* By default FW has this off for performance reasons */
9043         pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
9044
9045         /* set up queue assignment tracking */
9046         size = sizeof(struct i40e_lump_tracking)
9047                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
9048         pf->qp_pile = kzalloc(size, GFP_KERNEL);
9049         if (!pf->qp_pile) {
9050                 err = -ENOMEM;
9051                 goto sw_init_done;
9052         }
9053         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
9054         pf->qp_pile->search_hint = 0;
9055
9056         pf->tx_timeout_recovery_level = 1;
9057
9058         mutex_init(&pf->switch_mutex);
9059
9060 sw_init_done:
9061         return err;
9062 }
9063
9064 /**
9065  * i40e_set_ntuple - set the ntuple feature flag and take action
9066  * @pf: board private structure to initialize
9067  * @features: the feature set that the stack is suggesting
9068  *
9069  * returns a bool to indicate if reset needs to happen
9070  **/
9071 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
9072 {
9073         bool need_reset = false;
9074
9075         /* Check if Flow Director n-tuple support was enabled or disabled.  If
9076          * the state changed, we need to reset.
9077          */
9078         if (features & NETIF_F_NTUPLE) {
9079                 /* Enable filters and mark for reset */
9080                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
9081                         need_reset = true;
9082                 /* enable FD_SB only if there is MSI-X vector */
9083                 if (pf->num_fdsb_msix > 0)
9084                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
9085         } else {
9086                 /* turn off filters, mark for reset and clear SW filter list */
9087                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
9088                         need_reset = true;
9089                         i40e_fdir_filter_exit(pf);
9090                 }
9091                 pf->flags &= ~(I40E_FLAG_FD_SB_ENABLED |
9092                                I40E_FLAG_FD_SB_AUTO_DISABLED);
9093                 /* reset fd counters */
9094                 pf->fd_add_err = 0;
9095                 pf->fd_atr_cnt = 0;
9096                 /* if ATR was auto disabled it can be re-enabled. */
9097                 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) {
9098                         pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
9099                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
9100                             (I40E_DEBUG_FD & pf->hw.debug_mask))
9101                                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
9102                 }
9103         }
9104         return need_reset;
9105 }
9106
9107 /**
9108  * i40e_clear_rss_lut - clear the rx hash lookup table
9109  * @vsi: the VSI being configured
9110  **/
9111 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
9112 {
9113         struct i40e_pf *pf = vsi->back;
9114         struct i40e_hw *hw = &pf->hw;
9115         u16 vf_id = vsi->vf_id;
9116         u8 i;
9117
9118         if (vsi->type == I40E_VSI_MAIN) {
9119                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
9120                         wr32(hw, I40E_PFQF_HLUT(i), 0);
9121         } else if (vsi->type == I40E_VSI_SRIOV) {
9122                 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
9123                         i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
9124         } else {
9125                 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
9126         }
9127 }
9128
9129 /**
9130  * i40e_set_features - set the netdev feature flags
9131  * @netdev: ptr to the netdev being adjusted
9132  * @features: the feature set that the stack is suggesting
9133  * Note: expects to be called while under rtnl_lock()
9134  **/
9135 static int i40e_set_features(struct net_device *netdev,
9136                              netdev_features_t features)
9137 {
9138         struct i40e_netdev_priv *np = netdev_priv(netdev);
9139         struct i40e_vsi *vsi = np->vsi;
9140         struct i40e_pf *pf = vsi->back;
9141         bool need_reset;
9142
9143         if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
9144                 i40e_pf_config_rss(pf);
9145         else if (!(features & NETIF_F_RXHASH) &&
9146                  netdev->features & NETIF_F_RXHASH)
9147                 i40e_clear_rss_lut(vsi);
9148
9149         if (features & NETIF_F_HW_VLAN_CTAG_RX)
9150                 i40e_vlan_stripping_enable(vsi);
9151         else
9152                 i40e_vlan_stripping_disable(vsi);
9153
9154         need_reset = i40e_set_ntuple(pf, features);
9155
9156         if (need_reset)
9157                 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true);
9158
9159         return 0;
9160 }
9161
9162 /**
9163  * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port
9164  * @pf: board private structure
9165  * @port: The UDP port to look up
9166  *
9167  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
9168  **/
9169 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port)
9170 {
9171         u8 i;
9172
9173         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
9174                 if (pf->udp_ports[i].port == port)
9175                         return i;
9176         }
9177
9178         return i;
9179 }
9180
9181 /**
9182  * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up
9183  * @netdev: This physical port's netdev
9184  * @ti: Tunnel endpoint information
9185  **/
9186 static void i40e_udp_tunnel_add(struct net_device *netdev,
9187                                 struct udp_tunnel_info *ti)
9188 {
9189         struct i40e_netdev_priv *np = netdev_priv(netdev);
9190         struct i40e_vsi *vsi = np->vsi;
9191         struct i40e_pf *pf = vsi->back;
9192         u16 port = ntohs(ti->port);
9193         u8 next_idx;
9194         u8 idx;
9195
9196         idx = i40e_get_udp_port_idx(pf, port);
9197
9198         /* Check if port already exists */
9199         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
9200                 netdev_info(netdev, "port %d already offloaded\n", port);
9201                 return;
9202         }
9203
9204         /* Now check if there is space to add the new port */
9205         next_idx = i40e_get_udp_port_idx(pf, 0);
9206
9207         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
9208                 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n",
9209                             port);
9210                 return;
9211         }
9212
9213         switch (ti->type) {
9214         case UDP_TUNNEL_TYPE_VXLAN:
9215                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
9216                 break;
9217         case UDP_TUNNEL_TYPE_GENEVE:
9218                 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE))
9219                         return;
9220                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
9221                 break;
9222         default:
9223                 return;
9224         }
9225
9226         /* New port: add it and mark its index in the bitmap */
9227         pf->udp_ports[next_idx].port = port;
9228         pf->pending_udp_bitmap |= BIT_ULL(next_idx);
9229         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
9230 }
9231
9232 /**
9233  * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away
9234  * @netdev: This physical port's netdev
9235  * @ti: Tunnel endpoint information
9236  **/
9237 static void i40e_udp_tunnel_del(struct net_device *netdev,
9238                                 struct udp_tunnel_info *ti)
9239 {
9240         struct i40e_netdev_priv *np = netdev_priv(netdev);
9241         struct i40e_vsi *vsi = np->vsi;
9242         struct i40e_pf *pf = vsi->back;
9243         u16 port = ntohs(ti->port);
9244         u8 idx;
9245
9246         idx = i40e_get_udp_port_idx(pf, port);
9247
9248         /* Check if port already exists */
9249         if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS)
9250                 goto not_found;
9251
9252         switch (ti->type) {
9253         case UDP_TUNNEL_TYPE_VXLAN:
9254                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN)
9255                         goto not_found;
9256                 break;
9257         case UDP_TUNNEL_TYPE_GENEVE:
9258                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE)
9259                         goto not_found;
9260                 break;
9261         default:
9262                 goto not_found;
9263         }
9264
9265         /* if port exists, set it to 0 (mark for deletion)
9266          * and make it pending
9267          */
9268         pf->udp_ports[idx].port = 0;
9269         pf->pending_udp_bitmap |= BIT_ULL(idx);
9270         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
9271
9272         return;
9273 not_found:
9274         netdev_warn(netdev, "UDP port %d was not found, not deleting\n",
9275                     port);
9276 }
9277
9278 static int i40e_get_phys_port_id(struct net_device *netdev,
9279                                  struct netdev_phys_item_id *ppid)
9280 {
9281         struct i40e_netdev_priv *np = netdev_priv(netdev);
9282         struct i40e_pf *pf = np->vsi->back;
9283         struct i40e_hw *hw = &pf->hw;
9284
9285         if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
9286                 return -EOPNOTSUPP;
9287
9288         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
9289         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
9290
9291         return 0;
9292 }
9293
9294 /**
9295  * i40e_ndo_fdb_add - add an entry to the hardware database
9296  * @ndm: the input from the stack
9297  * @tb: pointer to array of nladdr (unused)
9298  * @dev: the net device pointer
9299  * @addr: the MAC address entry being added
9300  * @flags: instructions from stack about fdb operation
9301  */
9302 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
9303                             struct net_device *dev,
9304                             const unsigned char *addr, u16 vid,
9305                             u16 flags)
9306 {
9307         struct i40e_netdev_priv *np = netdev_priv(dev);
9308         struct i40e_pf *pf = np->vsi->back;
9309         int err = 0;
9310
9311         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
9312                 return -EOPNOTSUPP;
9313
9314         if (vid) {
9315                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
9316                 return -EINVAL;
9317         }
9318
9319         /* Hardware does not support aging addresses so if a
9320          * ndm_state is given only allow permanent addresses
9321          */
9322         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
9323                 netdev_info(dev, "FDB only supports static addresses\n");
9324                 return -EINVAL;
9325         }
9326
9327         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
9328                 err = dev_uc_add_excl(dev, addr);
9329         else if (is_multicast_ether_addr(addr))
9330                 err = dev_mc_add_excl(dev, addr);
9331         else
9332                 err = -EINVAL;
9333
9334         /* Only return duplicate errors if NLM_F_EXCL is set */
9335         if (err == -EEXIST && !(flags & NLM_F_EXCL))
9336                 err = 0;
9337
9338         return err;
9339 }
9340
9341 /**
9342  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
9343  * @dev: the netdev being configured
9344  * @nlh: RTNL message
9345  *
9346  * Inserts a new hardware bridge if not already created and
9347  * enables the bridging mode requested (VEB or VEPA). If the
9348  * hardware bridge has already been inserted and the request
9349  * is to change the mode then that requires a PF reset to
9350  * allow rebuild of the components with required hardware
9351  * bridge mode enabled.
9352  *
9353  * Note: expects to be called while under rtnl_lock()
9354  **/
9355 static int i40e_ndo_bridge_setlink(struct net_device *dev,
9356                                    struct nlmsghdr *nlh,
9357                                    u16 flags)
9358 {
9359         struct i40e_netdev_priv *np = netdev_priv(dev);
9360         struct i40e_vsi *vsi = np->vsi;
9361         struct i40e_pf *pf = vsi->back;
9362         struct i40e_veb *veb = NULL;
9363         struct nlattr *attr, *br_spec;
9364         int i, rem;
9365
9366         /* Only for PF VSI for now */
9367         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
9368                 return -EOPNOTSUPP;
9369
9370         /* Find the HW bridge for PF VSI */
9371         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9372                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9373                         veb = pf->veb[i];
9374         }
9375
9376         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
9377
9378         nla_for_each_nested(attr, br_spec, rem) {
9379                 __u16 mode;
9380
9381                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
9382                         continue;
9383
9384                 mode = nla_get_u16(attr);
9385                 if ((mode != BRIDGE_MODE_VEPA) &&
9386                     (mode != BRIDGE_MODE_VEB))
9387                         return -EINVAL;
9388
9389                 /* Insert a new HW bridge */
9390                 if (!veb) {
9391                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
9392                                              vsi->tc_config.enabled_tc);
9393                         if (veb) {
9394                                 veb->bridge_mode = mode;
9395                                 i40e_config_bridge_mode(veb);
9396                         } else {
9397                                 /* No Bridge HW offload available */
9398                                 return -ENOENT;
9399                         }
9400                         break;
9401                 } else if (mode != veb->bridge_mode) {
9402                         /* Existing HW bridge but different mode needs reset */
9403                         veb->bridge_mode = mode;
9404                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
9405                         if (mode == BRIDGE_MODE_VEB)
9406                                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
9407                         else
9408                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
9409                         i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED),
9410                                       true);
9411                         break;
9412                 }
9413         }
9414
9415         return 0;
9416 }
9417
9418 /**
9419  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
9420  * @skb: skb buff
9421  * @pid: process id
9422  * @seq: RTNL message seq #
9423  * @dev: the netdev being configured
9424  * @filter_mask: unused
9425  * @nlflags: netlink flags passed in
9426  *
9427  * Return the mode in which the hardware bridge is operating in
9428  * i.e VEB or VEPA.
9429  **/
9430 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
9431                                    struct net_device *dev,
9432                                    u32 __always_unused filter_mask,
9433                                    int nlflags)
9434 {
9435         struct i40e_netdev_priv *np = netdev_priv(dev);
9436         struct i40e_vsi *vsi = np->vsi;
9437         struct i40e_pf *pf = vsi->back;
9438         struct i40e_veb *veb = NULL;
9439         int i;
9440
9441         /* Only for PF VSI for now */
9442         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
9443                 return -EOPNOTSUPP;
9444
9445         /* Find the HW bridge for the PF VSI */
9446         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9447                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9448                         veb = pf->veb[i];
9449         }
9450
9451         if (!veb)
9452                 return 0;
9453
9454         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
9455                                        0, 0, nlflags, filter_mask, NULL);
9456 }
9457
9458 /**
9459  * i40e_features_check - Validate encapsulated packet conforms to limits
9460  * @skb: skb buff
9461  * @dev: This physical port's netdev
9462  * @features: Offload features that the stack believes apply
9463  **/
9464 static netdev_features_t i40e_features_check(struct sk_buff *skb,
9465                                              struct net_device *dev,
9466                                              netdev_features_t features)
9467 {
9468         size_t len;
9469
9470         /* No point in doing any of this if neither checksum nor GSO are
9471          * being requested for this frame.  We can rule out both by just
9472          * checking for CHECKSUM_PARTIAL
9473          */
9474         if (skb->ip_summed != CHECKSUM_PARTIAL)
9475                 return features;
9476
9477         /* We cannot support GSO if the MSS is going to be less than
9478          * 64 bytes.  If it is then we need to drop support for GSO.
9479          */
9480         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
9481                 features &= ~NETIF_F_GSO_MASK;
9482
9483         /* MACLEN can support at most 63 words */
9484         len = skb_network_header(skb) - skb->data;
9485         if (len & ~(63 * 2))
9486                 goto out_err;
9487
9488         /* IPLEN and EIPLEN can support at most 127 dwords */
9489         len = skb_transport_header(skb) - skb_network_header(skb);
9490         if (len & ~(127 * 4))
9491                 goto out_err;
9492
9493         if (skb->encapsulation) {
9494                 /* L4TUNLEN can support 127 words */
9495                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
9496                 if (len & ~(127 * 2))
9497                         goto out_err;
9498
9499                 /* IPLEN can support at most 127 dwords */
9500                 len = skb_inner_transport_header(skb) -
9501                       skb_inner_network_header(skb);
9502                 if (len & ~(127 * 4))
9503                         goto out_err;
9504         }
9505
9506         /* No need to validate L4LEN as TCP is the only protocol with a
9507          * a flexible value and we support all possible values supported
9508          * by TCP, which is at most 15 dwords
9509          */
9510
9511         return features;
9512 out_err:
9513         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
9514 }
9515
9516 /**
9517  * i40e_xdp_setup - add/remove an XDP program
9518  * @vsi: VSI to changed
9519  * @prog: XDP program
9520  **/
9521 static int i40e_xdp_setup(struct i40e_vsi *vsi,
9522                           struct bpf_prog *prog)
9523 {
9524         int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
9525         struct i40e_pf *pf = vsi->back;
9526         struct bpf_prog *old_prog;
9527         bool need_reset;
9528         int i;
9529
9530         /* Don't allow frames that span over multiple buffers */
9531         if (frame_size > vsi->rx_buf_len)
9532                 return -EINVAL;
9533
9534         if (!i40e_enabled_xdp_vsi(vsi) && !prog)
9535                 return 0;
9536
9537         /* When turning XDP on->off/off->on we reset and rebuild the rings. */
9538         need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
9539
9540         if (need_reset)
9541                 i40e_prep_for_reset(pf, true);
9542
9543         old_prog = xchg(&vsi->xdp_prog, prog);
9544
9545         if (need_reset)
9546                 i40e_reset_and_rebuild(pf, true, true);
9547
9548         for (i = 0; i < vsi->num_queue_pairs; i++)
9549                 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
9550
9551         if (old_prog)
9552                 bpf_prog_put(old_prog);
9553
9554         return 0;
9555 }
9556
9557 /**
9558  * i40e_xdp - implements ndo_xdp for i40e
9559  * @dev: netdevice
9560  * @xdp: XDP command
9561  **/
9562 static int i40e_xdp(struct net_device *dev,
9563                     struct netdev_xdp *xdp)
9564 {
9565         struct i40e_netdev_priv *np = netdev_priv(dev);
9566         struct i40e_vsi *vsi = np->vsi;
9567
9568         if (vsi->type != I40E_VSI_MAIN)
9569                 return -EINVAL;
9570
9571         switch (xdp->command) {
9572         case XDP_SETUP_PROG:
9573                 return i40e_xdp_setup(vsi, xdp->prog);
9574         case XDP_QUERY_PROG:
9575                 xdp->prog_attached = i40e_enabled_xdp_vsi(vsi);
9576                 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0;
9577                 return 0;
9578         default:
9579                 return -EINVAL;
9580         }
9581 }
9582
9583 static const struct net_device_ops i40e_netdev_ops = {
9584         .ndo_open               = i40e_open,
9585         .ndo_stop               = i40e_close,
9586         .ndo_start_xmit         = i40e_lan_xmit_frame,
9587         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
9588         .ndo_set_rx_mode        = i40e_set_rx_mode,
9589         .ndo_validate_addr      = eth_validate_addr,
9590         .ndo_set_mac_address    = i40e_set_mac,
9591         .ndo_change_mtu         = i40e_change_mtu,
9592         .ndo_do_ioctl           = i40e_ioctl,
9593         .ndo_tx_timeout         = i40e_tx_timeout,
9594         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
9595         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
9596 #ifdef CONFIG_NET_POLL_CONTROLLER
9597         .ndo_poll_controller    = i40e_netpoll,
9598 #endif
9599         .ndo_setup_tc           = __i40e_setup_tc,
9600         .ndo_set_features       = i40e_set_features,
9601         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
9602         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
9603         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
9604         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
9605         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
9606         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
9607         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
9608         .ndo_udp_tunnel_add     = i40e_udp_tunnel_add,
9609         .ndo_udp_tunnel_del     = i40e_udp_tunnel_del,
9610         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
9611         .ndo_fdb_add            = i40e_ndo_fdb_add,
9612         .ndo_features_check     = i40e_features_check,
9613         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
9614         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
9615         .ndo_xdp                = i40e_xdp,
9616 };
9617
9618 /**
9619  * i40e_config_netdev - Setup the netdev flags
9620  * @vsi: the VSI being configured
9621  *
9622  * Returns 0 on success, negative value on failure
9623  **/
9624 static int i40e_config_netdev(struct i40e_vsi *vsi)
9625 {
9626         struct i40e_pf *pf = vsi->back;
9627         struct i40e_hw *hw = &pf->hw;
9628         struct i40e_netdev_priv *np;
9629         struct net_device *netdev;
9630         u8 broadcast[ETH_ALEN];
9631         u8 mac_addr[ETH_ALEN];
9632         int etherdev_size;
9633         netdev_features_t hw_enc_features;
9634         netdev_features_t hw_features;
9635
9636         etherdev_size = sizeof(struct i40e_netdev_priv);
9637         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
9638         if (!netdev)
9639                 return -ENOMEM;
9640
9641         vsi->netdev = netdev;
9642         np = netdev_priv(netdev);
9643         np->vsi = vsi;
9644
9645         hw_enc_features = NETIF_F_SG                    |
9646                           NETIF_F_IP_CSUM               |
9647                           NETIF_F_IPV6_CSUM             |
9648                           NETIF_F_HIGHDMA               |
9649                           NETIF_F_SOFT_FEATURES         |
9650                           NETIF_F_TSO                   |
9651                           NETIF_F_TSO_ECN               |
9652                           NETIF_F_TSO6                  |
9653                           NETIF_F_GSO_GRE               |
9654                           NETIF_F_GSO_GRE_CSUM          |
9655                           NETIF_F_GSO_PARTIAL           |
9656                           NETIF_F_GSO_UDP_TUNNEL        |
9657                           NETIF_F_GSO_UDP_TUNNEL_CSUM   |
9658                           NETIF_F_SCTP_CRC              |
9659                           NETIF_F_RXHASH                |
9660                           NETIF_F_RXCSUM                |
9661                           0;
9662
9663         if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
9664                 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
9665
9666         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
9667
9668         netdev->hw_enc_features |= hw_enc_features;
9669
9670         /* record features VLANs can make use of */
9671         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
9672
9673         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
9674                 netdev->hw_features |= NETIF_F_NTUPLE;
9675         hw_features = hw_enc_features           |
9676                       NETIF_F_HW_VLAN_CTAG_TX   |
9677                       NETIF_F_HW_VLAN_CTAG_RX;
9678
9679         netdev->hw_features |= hw_features;
9680
9681         netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
9682         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
9683
9684         if (vsi->type == I40E_VSI_MAIN) {
9685                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
9686                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
9687                 /* The following steps are necessary for two reasons. First,
9688                  * some older NVM configurations load a default MAC-VLAN
9689                  * filter that will accept any tagged packet, and we want to
9690                  * replace this with a normal filter. Additionally, it is
9691                  * possible our MAC address was provided by the platform using
9692                  * Open Firmware or similar.
9693                  *
9694                  * Thus, we need to remove the default filter and install one
9695                  * specific to the MAC address.
9696                  */
9697                 i40e_rm_default_mac_filter(vsi, mac_addr);
9698                 spin_lock_bh(&vsi->mac_filter_hash_lock);
9699                 i40e_add_mac_filter(vsi, mac_addr);
9700                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9701         } else {
9702                 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
9703                  * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
9704                  * the end, which is 4 bytes long, so force truncation of the
9705                  * original name by IFNAMSIZ - 4
9706                  */
9707                 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
9708                          IFNAMSIZ - 4,
9709                          pf->vsi[pf->lan_vsi]->netdev->name);
9710                 random_ether_addr(mac_addr);
9711
9712                 spin_lock_bh(&vsi->mac_filter_hash_lock);
9713                 i40e_add_mac_filter(vsi, mac_addr);
9714                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9715         }
9716
9717         /* Add the broadcast filter so that we initially will receive
9718          * broadcast packets. Note that when a new VLAN is first added the
9719          * driver will convert all filters marked I40E_VLAN_ANY into VLAN
9720          * specific filters as part of transitioning into "vlan" operation.
9721          * When more VLANs are added, the driver will copy each existing MAC
9722          * filter and add it for the new VLAN.
9723          *
9724          * Broadcast filters are handled specially by
9725          * i40e_sync_filters_subtask, as the driver must to set the broadcast
9726          * promiscuous bit instead of adding this directly as a MAC/VLAN
9727          * filter. The subtask will update the correct broadcast promiscuous
9728          * bits as VLANs become active or inactive.
9729          */
9730         eth_broadcast_addr(broadcast);
9731         spin_lock_bh(&vsi->mac_filter_hash_lock);
9732         i40e_add_mac_filter(vsi, broadcast);
9733         spin_unlock_bh(&vsi->mac_filter_hash_lock);
9734
9735         ether_addr_copy(netdev->dev_addr, mac_addr);
9736         ether_addr_copy(netdev->perm_addr, mac_addr);
9737
9738         netdev->priv_flags |= IFF_UNICAST_FLT;
9739         netdev->priv_flags |= IFF_SUPP_NOFCS;
9740         /* Setup netdev TC information */
9741         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
9742
9743         netdev->netdev_ops = &i40e_netdev_ops;
9744         netdev->watchdog_timeo = 5 * HZ;
9745         i40e_set_ethtool_ops(netdev);
9746
9747         /* MTU range: 68 - 9706 */
9748         netdev->min_mtu = ETH_MIN_MTU;
9749         netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
9750
9751         return 0;
9752 }
9753
9754 /**
9755  * i40e_vsi_delete - Delete a VSI from the switch
9756  * @vsi: the VSI being removed
9757  *
9758  * Returns 0 on success, negative value on failure
9759  **/
9760 static void i40e_vsi_delete(struct i40e_vsi *vsi)
9761 {
9762         /* remove default VSI is not allowed */
9763         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
9764                 return;
9765
9766         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
9767 }
9768
9769 /**
9770  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
9771  * @vsi: the VSI being queried
9772  *
9773  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
9774  **/
9775 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
9776 {
9777         struct i40e_veb *veb;
9778         struct i40e_pf *pf = vsi->back;
9779
9780         /* Uplink is not a bridge so default to VEB */
9781         if (vsi->veb_idx == I40E_NO_VEB)
9782                 return 1;
9783
9784         veb = pf->veb[vsi->veb_idx];
9785         if (!veb) {
9786                 dev_info(&pf->pdev->dev,
9787                          "There is no veb associated with the bridge\n");
9788                 return -ENOENT;
9789         }
9790
9791         /* Uplink is a bridge in VEPA mode */
9792         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
9793                 return 0;
9794         } else {
9795                 /* Uplink is a bridge in VEB mode */
9796                 return 1;
9797         }
9798
9799         /* VEPA is now default bridge, so return 0 */
9800         return 0;
9801 }
9802
9803 /**
9804  * i40e_add_vsi - Add a VSI to the switch
9805  * @vsi: the VSI being configured
9806  *
9807  * This initializes a VSI context depending on the VSI type to be added and
9808  * passes it down to the add_vsi aq command.
9809  **/
9810 static int i40e_add_vsi(struct i40e_vsi *vsi)
9811 {
9812         int ret = -ENODEV;
9813         struct i40e_pf *pf = vsi->back;
9814         struct i40e_hw *hw = &pf->hw;
9815         struct i40e_vsi_context ctxt;
9816         struct i40e_mac_filter *f;
9817         struct hlist_node *h;
9818         int bkt;
9819
9820         u8 enabled_tc = 0x1; /* TC0 enabled */
9821         int f_count = 0;
9822
9823         memset(&ctxt, 0, sizeof(ctxt));
9824         switch (vsi->type) {
9825         case I40E_VSI_MAIN:
9826                 /* The PF's main VSI is already setup as part of the
9827                  * device initialization, so we'll not bother with
9828                  * the add_vsi call, but we will retrieve the current
9829                  * VSI context.
9830                  */
9831                 ctxt.seid = pf->main_vsi_seid;
9832                 ctxt.pf_num = pf->hw.pf_id;
9833                 ctxt.vf_num = 0;
9834                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9835                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9836                 if (ret) {
9837                         dev_info(&pf->pdev->dev,
9838                                  "couldn't get PF vsi config, err %s aq_err %s\n",
9839                                  i40e_stat_str(&pf->hw, ret),
9840                                  i40e_aq_str(&pf->hw,
9841                                              pf->hw.aq.asq_last_status));
9842                         return -ENOENT;
9843                 }
9844                 vsi->info = ctxt.info;
9845                 vsi->info.valid_sections = 0;
9846
9847                 vsi->seid = ctxt.seid;
9848                 vsi->id = ctxt.vsi_number;
9849
9850                 enabled_tc = i40e_pf_get_tc_map(pf);
9851
9852                 /* MFP mode setup queue map and update VSI */
9853                 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
9854                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
9855                         memset(&ctxt, 0, sizeof(ctxt));
9856                         ctxt.seid = pf->main_vsi_seid;
9857                         ctxt.pf_num = pf->hw.pf_id;
9858                         ctxt.vf_num = 0;
9859                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
9860                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
9861                         if (ret) {
9862                                 dev_info(&pf->pdev->dev,
9863                                          "update vsi failed, err %s aq_err %s\n",
9864                                          i40e_stat_str(&pf->hw, ret),
9865                                          i40e_aq_str(&pf->hw,
9866                                                     pf->hw.aq.asq_last_status));
9867                                 ret = -ENOENT;
9868                                 goto err;
9869                         }
9870                         /* update the local VSI info queue map */
9871                         i40e_vsi_update_queue_map(vsi, &ctxt);
9872                         vsi->info.valid_sections = 0;
9873                 } else {
9874                         /* Default/Main VSI is only enabled for TC0
9875                          * reconfigure it to enable all TCs that are
9876                          * available on the port in SFP mode.
9877                          * For MFP case the iSCSI PF would use this
9878                          * flow to enable LAN+iSCSI TC.
9879                          */
9880                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
9881                         if (ret) {
9882                                 /* Single TC condition is not fatal,
9883                                  * message and continue
9884                                  */
9885                                 dev_info(&pf->pdev->dev,
9886                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
9887                                          enabled_tc,
9888                                          i40e_stat_str(&pf->hw, ret),
9889                                          i40e_aq_str(&pf->hw,
9890                                                     pf->hw.aq.asq_last_status));
9891                         }
9892                 }
9893                 break;
9894
9895         case I40E_VSI_FDIR:
9896                 ctxt.pf_num = hw->pf_id;
9897                 ctxt.vf_num = 0;
9898                 ctxt.uplink_seid = vsi->uplink_seid;
9899                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9900                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9901                 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
9902                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
9903                         ctxt.info.valid_sections |=
9904                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9905                         ctxt.info.switch_id =
9906                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9907                 }
9908                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9909                 break;
9910
9911         case I40E_VSI_VMDQ2:
9912                 ctxt.pf_num = hw->pf_id;
9913                 ctxt.vf_num = 0;
9914                 ctxt.uplink_seid = vsi->uplink_seid;
9915                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9916                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
9917
9918                 /* This VSI is connected to VEB so the switch_id
9919                  * should be set to zero by default.
9920                  */
9921                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
9922                         ctxt.info.valid_sections |=
9923                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9924                         ctxt.info.switch_id =
9925                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9926                 }
9927
9928                 /* Setup the VSI tx/rx queue map for TC0 only for now */
9929                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9930                 break;
9931
9932         case I40E_VSI_SRIOV:
9933                 ctxt.pf_num = hw->pf_id;
9934                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
9935                 ctxt.uplink_seid = vsi->uplink_seid;
9936                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9937                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
9938
9939                 /* This VSI is connected to VEB so the switch_id
9940                  * should be set to zero by default.
9941                  */
9942                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
9943                         ctxt.info.valid_sections |=
9944                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9945                         ctxt.info.switch_id =
9946                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9947                 }
9948
9949                 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
9950                         ctxt.info.valid_sections |=
9951                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
9952                         ctxt.info.queueing_opt_flags |=
9953                                 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
9954                                  I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
9955                 }
9956
9957                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
9958                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
9959                 if (pf->vf[vsi->vf_id].spoofchk) {
9960                         ctxt.info.valid_sections |=
9961                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
9962                         ctxt.info.sec_flags |=
9963                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
9964                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
9965                 }
9966                 /* Setup the VSI tx/rx queue map for TC0 only for now */
9967                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9968                 break;
9969
9970         case I40E_VSI_IWARP:
9971                 /* send down message to iWARP */
9972                 break;
9973
9974         default:
9975                 return -ENODEV;
9976         }
9977
9978         if (vsi->type != I40E_VSI_MAIN) {
9979                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
9980                 if (ret) {
9981                         dev_info(&vsi->back->pdev->dev,
9982                                  "add vsi failed, err %s aq_err %s\n",
9983                                  i40e_stat_str(&pf->hw, ret),
9984                                  i40e_aq_str(&pf->hw,
9985                                              pf->hw.aq.asq_last_status));
9986                         ret = -ENOENT;
9987                         goto err;
9988                 }
9989                 vsi->info = ctxt.info;
9990                 vsi->info.valid_sections = 0;
9991                 vsi->seid = ctxt.seid;
9992                 vsi->id = ctxt.vsi_number;
9993         }
9994
9995         vsi->active_filters = 0;
9996         clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
9997         spin_lock_bh(&vsi->mac_filter_hash_lock);
9998         /* If macvlan filters already exist, force them to get loaded */
9999         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
10000                 f->state = I40E_FILTER_NEW;
10001                 f_count++;
10002         }
10003         spin_unlock_bh(&vsi->mac_filter_hash_lock);
10004
10005         if (f_count) {
10006                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
10007                 pf->flags |= I40E_FLAG_FILTER_SYNC;
10008         }
10009
10010         /* Update VSI BW information */
10011         ret = i40e_vsi_get_bw_info(vsi);
10012         if (ret) {
10013                 dev_info(&pf->pdev->dev,
10014                          "couldn't get vsi bw info, err %s aq_err %s\n",
10015                          i40e_stat_str(&pf->hw, ret),
10016                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10017                 /* VSI is already added so not tearing that up */
10018                 ret = 0;
10019         }
10020
10021 err:
10022         return ret;
10023 }
10024
10025 /**
10026  * i40e_vsi_release - Delete a VSI and free its resources
10027  * @vsi: the VSI being removed
10028  *
10029  * Returns 0 on success or < 0 on error
10030  **/
10031 int i40e_vsi_release(struct i40e_vsi *vsi)
10032 {
10033         struct i40e_mac_filter *f;
10034         struct hlist_node *h;
10035         struct i40e_veb *veb = NULL;
10036         struct i40e_pf *pf;
10037         u16 uplink_seid;
10038         int i, n, bkt;
10039
10040         pf = vsi->back;
10041
10042         /* release of a VEB-owner or last VSI is not allowed */
10043         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
10044                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
10045                          vsi->seid, vsi->uplink_seid);
10046                 return -ENODEV;
10047         }
10048         if (vsi == pf->vsi[pf->lan_vsi] &&
10049             !test_bit(__I40E_DOWN, pf->state)) {
10050                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
10051                 return -ENODEV;
10052         }
10053
10054         uplink_seid = vsi->uplink_seid;
10055         if (vsi->type != I40E_VSI_SRIOV) {
10056                 if (vsi->netdev_registered) {
10057                         vsi->netdev_registered = false;
10058                         if (vsi->netdev) {
10059                                 /* results in a call to i40e_close() */
10060                                 unregister_netdev(vsi->netdev);
10061                         }
10062                 } else {
10063                         i40e_vsi_close(vsi);
10064                 }
10065                 i40e_vsi_disable_irq(vsi);
10066         }
10067
10068         spin_lock_bh(&vsi->mac_filter_hash_lock);
10069
10070         /* clear the sync flag on all filters */
10071         if (vsi->netdev) {
10072                 __dev_uc_unsync(vsi->netdev, NULL);
10073                 __dev_mc_unsync(vsi->netdev, NULL);
10074         }
10075
10076         /* make sure any remaining filters are marked for deletion */
10077         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
10078                 __i40e_del_filter(vsi, f);
10079
10080         spin_unlock_bh(&vsi->mac_filter_hash_lock);
10081
10082         i40e_sync_vsi_filters(vsi);
10083
10084         i40e_vsi_delete(vsi);
10085         i40e_vsi_free_q_vectors(vsi);
10086         if (vsi->netdev) {
10087                 free_netdev(vsi->netdev);
10088                 vsi->netdev = NULL;
10089         }
10090         i40e_vsi_clear_rings(vsi);
10091         i40e_vsi_clear(vsi);
10092
10093         /* If this was the last thing on the VEB, except for the
10094          * controlling VSI, remove the VEB, which puts the controlling
10095          * VSI onto the next level down in the switch.
10096          *
10097          * Well, okay, there's one more exception here: don't remove
10098          * the orphan VEBs yet.  We'll wait for an explicit remove request
10099          * from up the network stack.
10100          */
10101         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
10102                 if (pf->vsi[i] &&
10103                     pf->vsi[i]->uplink_seid == uplink_seid &&
10104                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
10105                         n++;      /* count the VSIs */
10106                 }
10107         }
10108         for (i = 0; i < I40E_MAX_VEB; i++) {
10109                 if (!pf->veb[i])
10110                         continue;
10111                 if (pf->veb[i]->uplink_seid == uplink_seid)
10112                         n++;     /* count the VEBs */
10113                 if (pf->veb[i]->seid == uplink_seid)
10114                         veb = pf->veb[i];
10115         }
10116         if (n == 0 && veb && veb->uplink_seid != 0)
10117                 i40e_veb_release(veb);
10118
10119         return 0;
10120 }
10121
10122 /**
10123  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
10124  * @vsi: ptr to the VSI
10125  *
10126  * This should only be called after i40e_vsi_mem_alloc() which allocates the
10127  * corresponding SW VSI structure and initializes num_queue_pairs for the
10128  * newly allocated VSI.
10129  *
10130  * Returns 0 on success or negative on failure
10131  **/
10132 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
10133 {
10134         int ret = -ENOENT;
10135         struct i40e_pf *pf = vsi->back;
10136
10137         if (vsi->q_vectors[0]) {
10138                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
10139                          vsi->seid);
10140                 return -EEXIST;
10141         }
10142
10143         if (vsi->base_vector) {
10144                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
10145                          vsi->seid, vsi->base_vector);
10146                 return -EEXIST;
10147         }
10148
10149         ret = i40e_vsi_alloc_q_vectors(vsi);
10150         if (ret) {
10151                 dev_info(&pf->pdev->dev,
10152                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
10153                          vsi->num_q_vectors, vsi->seid, ret);
10154                 vsi->num_q_vectors = 0;
10155                 goto vector_setup_out;
10156         }
10157
10158         /* In Legacy mode, we do not have to get any other vector since we
10159          * piggyback on the misc/ICR0 for queue interrupts.
10160         */
10161         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
10162                 return ret;
10163         if (vsi->num_q_vectors)
10164                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
10165                                                  vsi->num_q_vectors, vsi->idx);
10166         if (vsi->base_vector < 0) {
10167                 dev_info(&pf->pdev->dev,
10168                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
10169                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
10170                 i40e_vsi_free_q_vectors(vsi);
10171                 ret = -ENOENT;
10172                 goto vector_setup_out;
10173         }
10174
10175 vector_setup_out:
10176         return ret;
10177 }
10178
10179 /**
10180  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
10181  * @vsi: pointer to the vsi.
10182  *
10183  * This re-allocates a vsi's queue resources.
10184  *
10185  * Returns pointer to the successfully allocated and configured VSI sw struct
10186  * on success, otherwise returns NULL on failure.
10187  **/
10188 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
10189 {
10190         u16 alloc_queue_pairs;
10191         struct i40e_pf *pf;
10192         u8 enabled_tc;
10193         int ret;
10194
10195         if (!vsi)
10196                 return NULL;
10197
10198         pf = vsi->back;
10199
10200         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
10201         i40e_vsi_clear_rings(vsi);
10202
10203         i40e_vsi_free_arrays(vsi, false);
10204         i40e_set_num_rings_in_vsi(vsi);
10205         ret = i40e_vsi_alloc_arrays(vsi, false);
10206         if (ret)
10207                 goto err_vsi;
10208
10209         alloc_queue_pairs = vsi->alloc_queue_pairs *
10210                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
10211
10212         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
10213         if (ret < 0) {
10214                 dev_info(&pf->pdev->dev,
10215                          "failed to get tracking for %d queues for VSI %d err %d\n",
10216                          alloc_queue_pairs, vsi->seid, ret);
10217                 goto err_vsi;
10218         }
10219         vsi->base_queue = ret;
10220
10221         /* Update the FW view of the VSI. Force a reset of TC and queue
10222          * layout configurations.
10223          */
10224         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
10225         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
10226         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
10227         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
10228         if (vsi->type == I40E_VSI_MAIN)
10229                 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
10230
10231         /* assign it some queues */
10232         ret = i40e_alloc_rings(vsi);
10233         if (ret)
10234                 goto err_rings;
10235
10236         /* map all of the rings to the q_vectors */
10237         i40e_vsi_map_rings_to_vectors(vsi);
10238         return vsi;
10239
10240 err_rings:
10241         i40e_vsi_free_q_vectors(vsi);
10242         if (vsi->netdev_registered) {
10243                 vsi->netdev_registered = false;
10244                 unregister_netdev(vsi->netdev);
10245                 free_netdev(vsi->netdev);
10246                 vsi->netdev = NULL;
10247         }
10248         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
10249 err_vsi:
10250         i40e_vsi_clear(vsi);
10251         return NULL;
10252 }
10253
10254 /**
10255  * i40e_vsi_setup - Set up a VSI by a given type
10256  * @pf: board private structure
10257  * @type: VSI type
10258  * @uplink_seid: the switch element to link to
10259  * @param1: usage depends upon VSI type. For VF types, indicates VF id
10260  *
10261  * This allocates the sw VSI structure and its queue resources, then add a VSI
10262  * to the identified VEB.
10263  *
10264  * Returns pointer to the successfully allocated and configure VSI sw struct on
10265  * success, otherwise returns NULL on failure.
10266  **/
10267 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
10268                                 u16 uplink_seid, u32 param1)
10269 {
10270         struct i40e_vsi *vsi = NULL;
10271         struct i40e_veb *veb = NULL;
10272         u16 alloc_queue_pairs;
10273         int ret, i;
10274         int v_idx;
10275
10276         /* The requested uplink_seid must be either
10277          *     - the PF's port seid
10278          *              no VEB is needed because this is the PF
10279          *              or this is a Flow Director special case VSI
10280          *     - seid of an existing VEB
10281          *     - seid of a VSI that owns an existing VEB
10282          *     - seid of a VSI that doesn't own a VEB
10283          *              a new VEB is created and the VSI becomes the owner
10284          *     - seid of the PF VSI, which is what creates the first VEB
10285          *              this is a special case of the previous
10286          *
10287          * Find which uplink_seid we were given and create a new VEB if needed
10288          */
10289         for (i = 0; i < I40E_MAX_VEB; i++) {
10290                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
10291                         veb = pf->veb[i];
10292                         break;
10293                 }
10294         }
10295
10296         if (!veb && uplink_seid != pf->mac_seid) {
10297
10298                 for (i = 0; i < pf->num_alloc_vsi; i++) {
10299                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
10300                                 vsi = pf->vsi[i];
10301                                 break;
10302                         }
10303                 }
10304                 if (!vsi) {
10305                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
10306                                  uplink_seid);
10307                         return NULL;
10308                 }
10309
10310                 if (vsi->uplink_seid == pf->mac_seid)
10311                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
10312                                              vsi->tc_config.enabled_tc);
10313                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
10314                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
10315                                              vsi->tc_config.enabled_tc);
10316                 if (veb) {
10317                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
10318                                 dev_info(&vsi->back->pdev->dev,
10319                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
10320                                 return NULL;
10321                         }
10322                         /* We come up by default in VEPA mode if SRIOV is not
10323                          * already enabled, in which case we can't force VEPA
10324                          * mode.
10325                          */
10326                         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
10327                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
10328                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
10329                         }
10330                         i40e_config_bridge_mode(veb);
10331                 }
10332                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
10333                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
10334                                 veb = pf->veb[i];
10335                 }
10336                 if (!veb) {
10337                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
10338                         return NULL;
10339                 }
10340
10341                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
10342                 uplink_seid = veb->seid;
10343         }
10344
10345         /* get vsi sw struct */
10346         v_idx = i40e_vsi_mem_alloc(pf, type);
10347         if (v_idx < 0)
10348                 goto err_alloc;
10349         vsi = pf->vsi[v_idx];
10350         if (!vsi)
10351                 goto err_alloc;
10352         vsi->type = type;
10353         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
10354
10355         if (type == I40E_VSI_MAIN)
10356                 pf->lan_vsi = v_idx;
10357         else if (type == I40E_VSI_SRIOV)
10358                 vsi->vf_id = param1;
10359         /* assign it some queues */
10360         alloc_queue_pairs = vsi->alloc_queue_pairs *
10361                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
10362
10363         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
10364         if (ret < 0) {
10365                 dev_info(&pf->pdev->dev,
10366                          "failed to get tracking for %d queues for VSI %d err=%d\n",
10367                          alloc_queue_pairs, vsi->seid, ret);
10368                 goto err_vsi;
10369         }
10370         vsi->base_queue = ret;
10371
10372         /* get a VSI from the hardware */
10373         vsi->uplink_seid = uplink_seid;
10374         ret = i40e_add_vsi(vsi);
10375         if (ret)
10376                 goto err_vsi;
10377
10378         switch (vsi->type) {
10379         /* setup the netdev if needed */
10380         case I40E_VSI_MAIN:
10381         case I40E_VSI_VMDQ2:
10382                 ret = i40e_config_netdev(vsi);
10383                 if (ret)
10384                         goto err_netdev;
10385                 ret = register_netdev(vsi->netdev);
10386                 if (ret)
10387                         goto err_netdev;
10388                 vsi->netdev_registered = true;
10389                 netif_carrier_off(vsi->netdev);
10390 #ifdef CONFIG_I40E_DCB
10391                 /* Setup DCB netlink interface */
10392                 i40e_dcbnl_setup(vsi);
10393 #endif /* CONFIG_I40E_DCB */
10394                 /* fall through */
10395
10396         case I40E_VSI_FDIR:
10397                 /* set up vectors and rings if needed */
10398                 ret = i40e_vsi_setup_vectors(vsi);
10399                 if (ret)
10400                         goto err_msix;
10401
10402                 ret = i40e_alloc_rings(vsi);
10403                 if (ret)
10404                         goto err_rings;
10405
10406                 /* map all of the rings to the q_vectors */
10407                 i40e_vsi_map_rings_to_vectors(vsi);
10408
10409                 i40e_vsi_reset_stats(vsi);
10410                 break;
10411
10412         default:
10413                 /* no netdev or rings for the other VSI types */
10414                 break;
10415         }
10416
10417         if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
10418             (vsi->type == I40E_VSI_VMDQ2)) {
10419                 ret = i40e_vsi_config_rss(vsi);
10420         }
10421         return vsi;
10422
10423 err_rings:
10424         i40e_vsi_free_q_vectors(vsi);
10425 err_msix:
10426         if (vsi->netdev_registered) {
10427                 vsi->netdev_registered = false;
10428                 unregister_netdev(vsi->netdev);
10429                 free_netdev(vsi->netdev);
10430                 vsi->netdev = NULL;
10431         }
10432 err_netdev:
10433         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
10434 err_vsi:
10435         i40e_vsi_clear(vsi);
10436 err_alloc:
10437         return NULL;
10438 }
10439
10440 /**
10441  * i40e_veb_get_bw_info - Query VEB BW information
10442  * @veb: the veb to query
10443  *
10444  * Query the Tx scheduler BW configuration data for given VEB
10445  **/
10446 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
10447 {
10448         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
10449         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
10450         struct i40e_pf *pf = veb->pf;
10451         struct i40e_hw *hw = &pf->hw;
10452         u32 tc_bw_max;
10453         int ret = 0;
10454         int i;
10455
10456         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
10457                                                   &bw_data, NULL);
10458         if (ret) {
10459                 dev_info(&pf->pdev->dev,
10460                          "query veb bw config failed, err %s aq_err %s\n",
10461                          i40e_stat_str(&pf->hw, ret),
10462                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
10463                 goto out;
10464         }
10465
10466         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
10467                                                    &ets_data, NULL);
10468         if (ret) {
10469                 dev_info(&pf->pdev->dev,
10470                          "query veb bw ets config failed, err %s aq_err %s\n",
10471                          i40e_stat_str(&pf->hw, ret),
10472                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
10473                 goto out;
10474         }
10475
10476         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
10477         veb->bw_max_quanta = ets_data.tc_bw_max;
10478         veb->is_abs_credits = bw_data.absolute_credits_enable;
10479         veb->enabled_tc = ets_data.tc_valid_bits;
10480         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
10481                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
10482         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
10483                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
10484                 veb->bw_tc_limit_credits[i] =
10485                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
10486                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
10487         }
10488
10489 out:
10490         return ret;
10491 }
10492
10493 /**
10494  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
10495  * @pf: board private structure
10496  *
10497  * On error: returns error code (negative)
10498  * On success: returns vsi index in PF (positive)
10499  **/
10500 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
10501 {
10502         int ret = -ENOENT;
10503         struct i40e_veb *veb;
10504         int i;
10505
10506         /* Need to protect the allocation of switch elements at the PF level */
10507         mutex_lock(&pf->switch_mutex);
10508
10509         /* VEB list may be fragmented if VEB creation/destruction has
10510          * been happening.  We can afford to do a quick scan to look
10511          * for any free slots in the list.
10512          *
10513          * find next empty veb slot, looping back around if necessary
10514          */
10515         i = 0;
10516         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
10517                 i++;
10518         if (i >= I40E_MAX_VEB) {
10519                 ret = -ENOMEM;
10520                 goto err_alloc_veb;  /* out of VEB slots! */
10521         }
10522
10523         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
10524         if (!veb) {
10525                 ret = -ENOMEM;
10526                 goto err_alloc_veb;
10527         }
10528         veb->pf = pf;
10529         veb->idx = i;
10530         veb->enabled_tc = 1;
10531
10532         pf->veb[i] = veb;
10533         ret = i;
10534 err_alloc_veb:
10535         mutex_unlock(&pf->switch_mutex);
10536         return ret;
10537 }
10538
10539 /**
10540  * i40e_switch_branch_release - Delete a branch of the switch tree
10541  * @branch: where to start deleting
10542  *
10543  * This uses recursion to find the tips of the branch to be
10544  * removed, deleting until we get back to and can delete this VEB.
10545  **/
10546 static void i40e_switch_branch_release(struct i40e_veb *branch)
10547 {
10548         struct i40e_pf *pf = branch->pf;
10549         u16 branch_seid = branch->seid;
10550         u16 veb_idx = branch->idx;
10551         int i;
10552
10553         /* release any VEBs on this VEB - RECURSION */
10554         for (i = 0; i < I40E_MAX_VEB; i++) {
10555                 if (!pf->veb[i])
10556                         continue;
10557                 if (pf->veb[i]->uplink_seid == branch->seid)
10558                         i40e_switch_branch_release(pf->veb[i]);
10559         }
10560
10561         /* Release the VSIs on this VEB, but not the owner VSI.
10562          *
10563          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
10564          *       the VEB itself, so don't use (*branch) after this loop.
10565          */
10566         for (i = 0; i < pf->num_alloc_vsi; i++) {
10567                 if (!pf->vsi[i])
10568                         continue;
10569                 if (pf->vsi[i]->uplink_seid == branch_seid &&
10570                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
10571                         i40e_vsi_release(pf->vsi[i]);
10572                 }
10573         }
10574
10575         /* There's one corner case where the VEB might not have been
10576          * removed, so double check it here and remove it if needed.
10577          * This case happens if the veb was created from the debugfs
10578          * commands and no VSIs were added to it.
10579          */
10580         if (pf->veb[veb_idx])
10581                 i40e_veb_release(pf->veb[veb_idx]);
10582 }
10583
10584 /**
10585  * i40e_veb_clear - remove veb struct
10586  * @veb: the veb to remove
10587  **/
10588 static void i40e_veb_clear(struct i40e_veb *veb)
10589 {
10590         if (!veb)
10591                 return;
10592
10593         if (veb->pf) {
10594                 struct i40e_pf *pf = veb->pf;
10595
10596                 mutex_lock(&pf->switch_mutex);
10597                 if (pf->veb[veb->idx] == veb)
10598                         pf->veb[veb->idx] = NULL;
10599                 mutex_unlock(&pf->switch_mutex);
10600         }
10601
10602         kfree(veb);
10603 }
10604
10605 /**
10606  * i40e_veb_release - Delete a VEB and free its resources
10607  * @veb: the VEB being removed
10608  **/
10609 void i40e_veb_release(struct i40e_veb *veb)
10610 {
10611         struct i40e_vsi *vsi = NULL;
10612         struct i40e_pf *pf;
10613         int i, n = 0;
10614
10615         pf = veb->pf;
10616
10617         /* find the remaining VSI and check for extras */
10618         for (i = 0; i < pf->num_alloc_vsi; i++) {
10619                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
10620                         n++;
10621                         vsi = pf->vsi[i];
10622                 }
10623         }
10624         if (n != 1) {
10625                 dev_info(&pf->pdev->dev,
10626                          "can't remove VEB %d with %d VSIs left\n",
10627                          veb->seid, n);
10628                 return;
10629         }
10630
10631         /* move the remaining VSI to uplink veb */
10632         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
10633         if (veb->uplink_seid) {
10634                 vsi->uplink_seid = veb->uplink_seid;
10635                 if (veb->uplink_seid == pf->mac_seid)
10636                         vsi->veb_idx = I40E_NO_VEB;
10637                 else
10638                         vsi->veb_idx = veb->veb_idx;
10639         } else {
10640                 /* floating VEB */
10641                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
10642                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
10643         }
10644
10645         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10646         i40e_veb_clear(veb);
10647 }
10648
10649 /**
10650  * i40e_add_veb - create the VEB in the switch
10651  * @veb: the VEB to be instantiated
10652  * @vsi: the controlling VSI
10653  **/
10654 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
10655 {
10656         struct i40e_pf *pf = veb->pf;
10657         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
10658         int ret;
10659
10660         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
10661                               veb->enabled_tc, false,
10662                               &veb->seid, enable_stats, NULL);
10663
10664         /* get a VEB from the hardware */
10665         if (ret) {
10666                 dev_info(&pf->pdev->dev,
10667                          "couldn't add VEB, err %s aq_err %s\n",
10668                          i40e_stat_str(&pf->hw, ret),
10669                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10670                 return -EPERM;
10671         }
10672
10673         /* get statistics counter */
10674         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
10675                                          &veb->stats_idx, NULL, NULL, NULL);
10676         if (ret) {
10677                 dev_info(&pf->pdev->dev,
10678                          "couldn't get VEB statistics idx, err %s aq_err %s\n",
10679                          i40e_stat_str(&pf->hw, ret),
10680                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10681                 return -EPERM;
10682         }
10683         ret = i40e_veb_get_bw_info(veb);
10684         if (ret) {
10685                 dev_info(&pf->pdev->dev,
10686                          "couldn't get VEB bw info, err %s aq_err %s\n",
10687                          i40e_stat_str(&pf->hw, ret),
10688                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10689                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10690                 return -ENOENT;
10691         }
10692
10693         vsi->uplink_seid = veb->seid;
10694         vsi->veb_idx = veb->idx;
10695         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
10696
10697         return 0;
10698 }
10699
10700 /**
10701  * i40e_veb_setup - Set up a VEB
10702  * @pf: board private structure
10703  * @flags: VEB setup flags
10704  * @uplink_seid: the switch element to link to
10705  * @vsi_seid: the initial VSI seid
10706  * @enabled_tc: Enabled TC bit-map
10707  *
10708  * This allocates the sw VEB structure and links it into the switch
10709  * It is possible and legal for this to be a duplicate of an already
10710  * existing VEB.  It is also possible for both uplink and vsi seids
10711  * to be zero, in order to create a floating VEB.
10712  *
10713  * Returns pointer to the successfully allocated VEB sw struct on
10714  * success, otherwise returns NULL on failure.
10715  **/
10716 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
10717                                 u16 uplink_seid, u16 vsi_seid,
10718                                 u8 enabled_tc)
10719 {
10720         struct i40e_veb *veb, *uplink_veb = NULL;
10721         int vsi_idx, veb_idx;
10722         int ret;
10723
10724         /* if one seid is 0, the other must be 0 to create a floating relay */
10725         if ((uplink_seid == 0 || vsi_seid == 0) &&
10726             (uplink_seid + vsi_seid != 0)) {
10727                 dev_info(&pf->pdev->dev,
10728                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
10729                          uplink_seid, vsi_seid);
10730                 return NULL;
10731         }
10732
10733         /* make sure there is such a vsi and uplink */
10734         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
10735                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
10736                         break;
10737         if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) {
10738                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
10739                          vsi_seid);
10740                 return NULL;
10741         }
10742
10743         if (uplink_seid && uplink_seid != pf->mac_seid) {
10744                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
10745                         if (pf->veb[veb_idx] &&
10746                             pf->veb[veb_idx]->seid == uplink_seid) {
10747                                 uplink_veb = pf->veb[veb_idx];
10748                                 break;
10749                         }
10750                 }
10751                 if (!uplink_veb) {
10752                         dev_info(&pf->pdev->dev,
10753                                  "uplink seid %d not found\n", uplink_seid);
10754                         return NULL;
10755                 }
10756         }
10757
10758         /* get veb sw struct */
10759         veb_idx = i40e_veb_mem_alloc(pf);
10760         if (veb_idx < 0)
10761                 goto err_alloc;
10762         veb = pf->veb[veb_idx];
10763         veb->flags = flags;
10764         veb->uplink_seid = uplink_seid;
10765         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
10766         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
10767
10768         /* create the VEB in the switch */
10769         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
10770         if (ret)
10771                 goto err_veb;
10772         if (vsi_idx == pf->lan_vsi)
10773                 pf->lan_veb = veb->idx;
10774
10775         return veb;
10776
10777 err_veb:
10778         i40e_veb_clear(veb);
10779 err_alloc:
10780         return NULL;
10781 }
10782
10783 /**
10784  * i40e_setup_pf_switch_element - set PF vars based on switch type
10785  * @pf: board private structure
10786  * @ele: element we are building info from
10787  * @num_reported: total number of elements
10788  * @printconfig: should we print the contents
10789  *
10790  * helper function to assist in extracting a few useful SEID values.
10791  **/
10792 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
10793                                 struct i40e_aqc_switch_config_element_resp *ele,
10794                                 u16 num_reported, bool printconfig)
10795 {
10796         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
10797         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
10798         u8 element_type = ele->element_type;
10799         u16 seid = le16_to_cpu(ele->seid);
10800
10801         if (printconfig)
10802                 dev_info(&pf->pdev->dev,
10803                          "type=%d seid=%d uplink=%d downlink=%d\n",
10804                          element_type, seid, uplink_seid, downlink_seid);
10805
10806         switch (element_type) {
10807         case I40E_SWITCH_ELEMENT_TYPE_MAC:
10808                 pf->mac_seid = seid;
10809                 break;
10810         case I40E_SWITCH_ELEMENT_TYPE_VEB:
10811                 /* Main VEB? */
10812                 if (uplink_seid != pf->mac_seid)
10813                         break;
10814                 if (pf->lan_veb == I40E_NO_VEB) {
10815                         int v;
10816
10817                         /* find existing or else empty VEB */
10818                         for (v = 0; v < I40E_MAX_VEB; v++) {
10819                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
10820                                         pf->lan_veb = v;
10821                                         break;
10822                                 }
10823                         }
10824                         if (pf->lan_veb == I40E_NO_VEB) {
10825                                 v = i40e_veb_mem_alloc(pf);
10826                                 if (v < 0)
10827                                         break;
10828                                 pf->lan_veb = v;
10829                         }
10830                 }
10831
10832                 pf->veb[pf->lan_veb]->seid = seid;
10833                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
10834                 pf->veb[pf->lan_veb]->pf = pf;
10835                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
10836                 break;
10837         case I40E_SWITCH_ELEMENT_TYPE_VSI:
10838                 if (num_reported != 1)
10839                         break;
10840                 /* This is immediately after a reset so we can assume this is
10841                  * the PF's VSI
10842                  */
10843                 pf->mac_seid = uplink_seid;
10844                 pf->pf_seid = downlink_seid;
10845                 pf->main_vsi_seid = seid;
10846                 if (printconfig)
10847                         dev_info(&pf->pdev->dev,
10848                                  "pf_seid=%d main_vsi_seid=%d\n",
10849                                  pf->pf_seid, pf->main_vsi_seid);
10850                 break;
10851         case I40E_SWITCH_ELEMENT_TYPE_PF:
10852         case I40E_SWITCH_ELEMENT_TYPE_VF:
10853         case I40E_SWITCH_ELEMENT_TYPE_EMP:
10854         case I40E_SWITCH_ELEMENT_TYPE_BMC:
10855         case I40E_SWITCH_ELEMENT_TYPE_PE:
10856         case I40E_SWITCH_ELEMENT_TYPE_PA:
10857                 /* ignore these for now */
10858                 break;
10859         default:
10860                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
10861                          element_type, seid);
10862                 break;
10863         }
10864 }
10865
10866 /**
10867  * i40e_fetch_switch_configuration - Get switch config from firmware
10868  * @pf: board private structure
10869  * @printconfig: should we print the contents
10870  *
10871  * Get the current switch configuration from the device and
10872  * extract a few useful SEID values.
10873  **/
10874 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
10875 {
10876         struct i40e_aqc_get_switch_config_resp *sw_config;
10877         u16 next_seid = 0;
10878         int ret = 0;
10879         u8 *aq_buf;
10880         int i;
10881
10882         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
10883         if (!aq_buf)
10884                 return -ENOMEM;
10885
10886         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
10887         do {
10888                 u16 num_reported, num_total;
10889
10890                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
10891                                                 I40E_AQ_LARGE_BUF,
10892                                                 &next_seid, NULL);
10893                 if (ret) {
10894                         dev_info(&pf->pdev->dev,
10895                                  "get switch config failed err %s aq_err %s\n",
10896                                  i40e_stat_str(&pf->hw, ret),
10897                                  i40e_aq_str(&pf->hw,
10898                                              pf->hw.aq.asq_last_status));
10899                         kfree(aq_buf);
10900                         return -ENOENT;
10901                 }
10902
10903                 num_reported = le16_to_cpu(sw_config->header.num_reported);
10904                 num_total = le16_to_cpu(sw_config->header.num_total);
10905
10906                 if (printconfig)
10907                         dev_info(&pf->pdev->dev,
10908                                  "header: %d reported %d total\n",
10909                                  num_reported, num_total);
10910
10911                 for (i = 0; i < num_reported; i++) {
10912                         struct i40e_aqc_switch_config_element_resp *ele =
10913                                 &sw_config->element[i];
10914
10915                         i40e_setup_pf_switch_element(pf, ele, num_reported,
10916                                                      printconfig);
10917                 }
10918         } while (next_seid != 0);
10919
10920         kfree(aq_buf);
10921         return ret;
10922 }
10923
10924 /**
10925  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
10926  * @pf: board private structure
10927  * @reinit: if the Main VSI needs to re-initialized.
10928  *
10929  * Returns 0 on success, negative value on failure
10930  **/
10931 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
10932 {
10933         u16 flags = 0;
10934         int ret;
10935
10936         /* find out what's out there already */
10937         ret = i40e_fetch_switch_configuration(pf, false);
10938         if (ret) {
10939                 dev_info(&pf->pdev->dev,
10940                          "couldn't fetch switch config, err %s aq_err %s\n",
10941                          i40e_stat_str(&pf->hw, ret),
10942                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10943                 return ret;
10944         }
10945         i40e_pf_reset_stats(pf);
10946
10947         /* set the switch config bit for the whole device to
10948          * support limited promisc or true promisc
10949          * when user requests promisc. The default is limited
10950          * promisc.
10951         */
10952
10953         if ((pf->hw.pf_id == 0) &&
10954             !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT))
10955                 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
10956
10957         if (pf->hw.pf_id == 0) {
10958                 u16 valid_flags;
10959
10960                 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
10961                 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags,
10962                                                 NULL);
10963                 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
10964                         dev_info(&pf->pdev->dev,
10965                                  "couldn't set switch config bits, err %s aq_err %s\n",
10966                                  i40e_stat_str(&pf->hw, ret),
10967                                  i40e_aq_str(&pf->hw,
10968                                              pf->hw.aq.asq_last_status));
10969                         /* not a fatal problem, just keep going */
10970                 }
10971         }
10972
10973         /* first time setup */
10974         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
10975                 struct i40e_vsi *vsi = NULL;
10976                 u16 uplink_seid;
10977
10978                 /* Set up the PF VSI associated with the PF's main VSI
10979                  * that is already in the HW switch
10980                  */
10981                 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
10982                         uplink_seid = pf->veb[pf->lan_veb]->seid;
10983                 else
10984                         uplink_seid = pf->mac_seid;
10985                 if (pf->lan_vsi == I40E_NO_VSI)
10986                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
10987                 else if (reinit)
10988                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
10989                 if (!vsi) {
10990                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
10991                         i40e_fdir_teardown(pf);
10992                         return -EAGAIN;
10993                 }
10994         } else {
10995                 /* force a reset of TC and queue layout configurations */
10996                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
10997
10998                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
10999                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
11000                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
11001         }
11002         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
11003
11004         i40e_fdir_sb_setup(pf);
11005
11006         /* Setup static PF queue filter control settings */
11007         ret = i40e_setup_pf_filter_control(pf);
11008         if (ret) {
11009                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
11010                          ret);
11011                 /* Failure here should not stop continuing other steps */
11012         }
11013
11014         /* enable RSS in the HW, even for only one queue, as the stack can use
11015          * the hash
11016          */
11017         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
11018                 i40e_pf_config_rss(pf);
11019
11020         /* fill in link information and enable LSE reporting */
11021         i40e_link_event(pf);
11022
11023         /* Initialize user-specific link properties */
11024         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
11025                                   I40E_AQ_AN_COMPLETED) ? true : false);
11026
11027         i40e_ptp_init(pf);
11028
11029         /* repopulate tunnel port filters */
11030         i40e_sync_udp_filters(pf);
11031
11032         return ret;
11033 }
11034
11035 /**
11036  * i40e_determine_queue_usage - Work out queue distribution
11037  * @pf: board private structure
11038  **/
11039 static void i40e_determine_queue_usage(struct i40e_pf *pf)
11040 {
11041         int queues_left;
11042
11043         pf->num_lan_qps = 0;
11044
11045         /* Find the max queues to be put into basic use.  We'll always be
11046          * using TC0, whether or not DCB is running, and TC0 will get the
11047          * big RSS set.
11048          */
11049         queues_left = pf->hw.func_caps.num_tx_qp;
11050
11051         if ((queues_left == 1) ||
11052             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
11053                 /* one qp for PF, no queues for anything else */
11054                 queues_left = 0;
11055                 pf->alloc_rss_size = pf->num_lan_qps = 1;
11056
11057                 /* make sure all the fancies are disabled */
11058                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
11059                                I40E_FLAG_IWARP_ENABLED  |
11060                                I40E_FLAG_FD_SB_ENABLED  |
11061                                I40E_FLAG_FD_ATR_ENABLED |
11062                                I40E_FLAG_DCB_CAPABLE    |
11063                                I40E_FLAG_DCB_ENABLED    |
11064                                I40E_FLAG_SRIOV_ENABLED  |
11065                                I40E_FLAG_VMDQ_ENABLED);
11066         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
11067                                   I40E_FLAG_FD_SB_ENABLED |
11068                                   I40E_FLAG_FD_ATR_ENABLED |
11069                                   I40E_FLAG_DCB_CAPABLE))) {
11070                 /* one qp for PF */
11071                 pf->alloc_rss_size = pf->num_lan_qps = 1;
11072                 queues_left -= pf->num_lan_qps;
11073
11074                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
11075                                I40E_FLAG_IWARP_ENABLED  |
11076                                I40E_FLAG_FD_SB_ENABLED  |
11077                                I40E_FLAG_FD_ATR_ENABLED |
11078                                I40E_FLAG_DCB_ENABLED    |
11079                                I40E_FLAG_VMDQ_ENABLED);
11080         } else {
11081                 /* Not enough queues for all TCs */
11082                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
11083                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
11084                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
11085                                         I40E_FLAG_DCB_ENABLED);
11086                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
11087                 }
11088                 pf->num_lan_qps = max_t(int, pf->rss_size_max,
11089                                         num_online_cpus());
11090                 pf->num_lan_qps = min_t(int, pf->num_lan_qps,
11091                                         pf->hw.func_caps.num_tx_qp);
11092
11093                 queues_left -= pf->num_lan_qps;
11094         }
11095
11096         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11097                 if (queues_left > 1) {
11098                         queues_left -= 1; /* save 1 queue for FD */
11099                 } else {
11100                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
11101                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
11102                 }
11103         }
11104
11105         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11106             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
11107                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
11108                                         (queues_left / pf->num_vf_qps));
11109                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
11110         }
11111
11112         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
11113             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
11114                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
11115                                           (queues_left / pf->num_vmdq_qps));
11116                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
11117         }
11118
11119         pf->queues_left = queues_left;
11120         dev_dbg(&pf->pdev->dev,
11121                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
11122                 pf->hw.func_caps.num_tx_qp,
11123                 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
11124                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
11125                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
11126                 queues_left);
11127 }
11128
11129 /**
11130  * i40e_setup_pf_filter_control - Setup PF static filter control
11131  * @pf: PF to be setup
11132  *
11133  * i40e_setup_pf_filter_control sets up a PF's initial filter control
11134  * settings. If PE/FCoE are enabled then it will also set the per PF
11135  * based filter sizes required for them. It also enables Flow director,
11136  * ethertype and macvlan type filter settings for the pf.
11137  *
11138  * Returns 0 on success, negative on failure
11139  **/
11140 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
11141 {
11142         struct i40e_filter_control_settings *settings = &pf->filter_settings;
11143
11144         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
11145
11146         /* Flow Director is enabled */
11147         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
11148                 settings->enable_fdir = true;
11149
11150         /* Ethtype and MACVLAN filters enabled for PF */
11151         settings->enable_ethtype = true;
11152         settings->enable_macvlan = true;
11153
11154         if (i40e_set_filter_control(&pf->hw, settings))
11155                 return -ENOENT;
11156
11157         return 0;
11158 }
11159
11160 #define INFO_STRING_LEN 255
11161 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
11162 static void i40e_print_features(struct i40e_pf *pf)
11163 {
11164         struct i40e_hw *hw = &pf->hw;
11165         char *buf;
11166         int i;
11167
11168         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
11169         if (!buf)
11170                 return;
11171
11172         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
11173 #ifdef CONFIG_PCI_IOV
11174         i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
11175 #endif
11176         i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
11177                       pf->hw.func_caps.num_vsis,
11178                       pf->vsi[pf->lan_vsi]->num_queue_pairs);
11179         if (pf->flags & I40E_FLAG_RSS_ENABLED)
11180                 i += snprintf(&buf[i], REMAIN(i), " RSS");
11181         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
11182                 i += snprintf(&buf[i], REMAIN(i), " FD_ATR");
11183         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11184                 i += snprintf(&buf[i], REMAIN(i), " FD_SB");
11185                 i += snprintf(&buf[i], REMAIN(i), " NTUPLE");
11186         }
11187         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
11188                 i += snprintf(&buf[i], REMAIN(i), " DCB");
11189         i += snprintf(&buf[i], REMAIN(i), " VxLAN");
11190         i += snprintf(&buf[i], REMAIN(i), " Geneve");
11191         if (pf->flags & I40E_FLAG_PTP)
11192                 i += snprintf(&buf[i], REMAIN(i), " PTP");
11193         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
11194                 i += snprintf(&buf[i], REMAIN(i), " VEB");
11195         else
11196                 i += snprintf(&buf[i], REMAIN(i), " VEPA");
11197
11198         dev_info(&pf->pdev->dev, "%s\n", buf);
11199         kfree(buf);
11200         WARN_ON(i > INFO_STRING_LEN);
11201 }
11202
11203 /**
11204  * i40e_get_platform_mac_addr - get platform-specific MAC address
11205  * @pdev: PCI device information struct
11206  * @pf: board private structure
11207  *
11208  * Look up the MAC address for the device. First we'll try
11209  * eth_platform_get_mac_address, which will check Open Firmware, or arch
11210  * specific fallback. Otherwise, we'll default to the stored value in
11211  * firmware.
11212  **/
11213 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
11214 {
11215         if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
11216                 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
11217 }
11218
11219 /**
11220  * i40e_probe - Device initialization routine
11221  * @pdev: PCI device information struct
11222  * @ent: entry in i40e_pci_tbl
11223  *
11224  * i40e_probe initializes a PF identified by a pci_dev structure.
11225  * The OS initialization, configuring of the PF private structure,
11226  * and a hardware reset occur.
11227  *
11228  * Returns 0 on success, negative on failure
11229  **/
11230 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
11231 {
11232         struct i40e_aq_get_phy_abilities_resp abilities;
11233         struct i40e_pf *pf;
11234         struct i40e_hw *hw;
11235         static u16 pfs_found;
11236         u16 wol_nvm_bits;
11237         u16 link_status;
11238         int err;
11239         u32 val;
11240         u32 i;
11241         u8 set_fc_aq_fail;
11242
11243         err = pci_enable_device_mem(pdev);
11244         if (err)
11245                 return err;
11246
11247         /* set up for high or low dma */
11248         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
11249         if (err) {
11250                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
11251                 if (err) {
11252                         dev_err(&pdev->dev,
11253                                 "DMA configuration failed: 0x%x\n", err);
11254                         goto err_dma;
11255                 }
11256         }
11257
11258         /* set up pci connections */
11259         err = pci_request_mem_regions(pdev, i40e_driver_name);
11260         if (err) {
11261                 dev_info(&pdev->dev,
11262                          "pci_request_selected_regions failed %d\n", err);
11263                 goto err_pci_reg;
11264         }
11265
11266         pci_enable_pcie_error_reporting(pdev);
11267         pci_set_master(pdev);
11268
11269         /* Now that we have a PCI connection, we need to do the
11270          * low level device setup.  This is primarily setting up
11271          * the Admin Queue structures and then querying for the
11272          * device's current profile information.
11273          */
11274         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
11275         if (!pf) {
11276                 err = -ENOMEM;
11277                 goto err_pf_alloc;
11278         }
11279         pf->next_vsi = 0;
11280         pf->pdev = pdev;
11281         set_bit(__I40E_DOWN, pf->state);
11282
11283         hw = &pf->hw;
11284         hw->back = pf;
11285
11286         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
11287                                 I40E_MAX_CSR_SPACE);
11288
11289         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
11290         if (!hw->hw_addr) {
11291                 err = -EIO;
11292                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
11293                          (unsigned int)pci_resource_start(pdev, 0),
11294                          pf->ioremap_len, err);
11295                 goto err_ioremap;
11296         }
11297         hw->vendor_id = pdev->vendor;
11298         hw->device_id = pdev->device;
11299         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
11300         hw->subsystem_vendor_id = pdev->subsystem_vendor;
11301         hw->subsystem_device_id = pdev->subsystem_device;
11302         hw->bus.device = PCI_SLOT(pdev->devfn);
11303         hw->bus.func = PCI_FUNC(pdev->devfn);
11304         hw->bus.bus_id = pdev->bus->number;
11305         pf->instance = pfs_found;
11306
11307         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
11308         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
11309
11310         /* set up the locks for the AQ, do this only once in probe
11311          * and destroy them only once in remove
11312          */
11313         mutex_init(&hw->aq.asq_mutex);
11314         mutex_init(&hw->aq.arq_mutex);
11315
11316         pf->msg_enable = netif_msg_init(debug,
11317                                         NETIF_MSG_DRV |
11318                                         NETIF_MSG_PROBE |
11319                                         NETIF_MSG_LINK);
11320         if (debug < -1)
11321                 pf->hw.debug_mask = debug;
11322
11323         /* do a special CORER for clearing PXE mode once at init */
11324         if (hw->revision_id == 0 &&
11325             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
11326                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
11327                 i40e_flush(hw);
11328                 msleep(200);
11329                 pf->corer_count++;
11330
11331                 i40e_clear_pxe_mode(hw);
11332         }
11333
11334         /* Reset here to make sure all is clean and to define PF 'n' */
11335         i40e_clear_hw(hw);
11336         err = i40e_pf_reset(hw);
11337         if (err) {
11338                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
11339                 goto err_pf_reset;
11340         }
11341         pf->pfr_count++;
11342
11343         hw->aq.num_arq_entries = I40E_AQ_LEN;
11344         hw->aq.num_asq_entries = I40E_AQ_LEN;
11345         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
11346         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
11347         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
11348
11349         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
11350                  "%s-%s:misc",
11351                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
11352
11353         err = i40e_init_shared_code(hw);
11354         if (err) {
11355                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
11356                          err);
11357                 goto err_pf_reset;
11358         }
11359
11360         /* set up a default setting for link flow control */
11361         pf->hw.fc.requested_mode = I40E_FC_NONE;
11362
11363         err = i40e_init_adminq(hw);
11364         if (err) {
11365                 if (err == I40E_ERR_FIRMWARE_API_VERSION)
11366                         dev_info(&pdev->dev,
11367                                  "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
11368                 else
11369                         dev_info(&pdev->dev,
11370                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
11371
11372                 goto err_pf_reset;
11373         }
11374         i40e_get_oem_version(hw);
11375
11376         /* provide nvm, fw, api versions */
11377         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n",
11378                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
11379                  hw->aq.api_maj_ver, hw->aq.api_min_ver,
11380                  i40e_nvm_version_str(hw));
11381
11382         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
11383             hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
11384                 dev_info(&pdev->dev,
11385                          "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
11386         else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
11387                  hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
11388                 dev_info(&pdev->dev,
11389                          "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
11390
11391         i40e_verify_eeprom(pf);
11392
11393         /* Rev 0 hardware was never productized */
11394         if (hw->revision_id < 1)
11395                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
11396
11397         i40e_clear_pxe_mode(hw);
11398         err = i40e_get_capabilities(pf);
11399         if (err)
11400                 goto err_adminq_setup;
11401
11402         err = i40e_sw_init(pf);
11403         if (err) {
11404                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
11405                 goto err_sw_init;
11406         }
11407
11408         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
11409                                 hw->func_caps.num_rx_qp, 0, 0);
11410         if (err) {
11411                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
11412                 goto err_init_lan_hmc;
11413         }
11414
11415         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
11416         if (err) {
11417                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
11418                 err = -ENOENT;
11419                 goto err_configure_lan_hmc;
11420         }
11421
11422         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
11423          * Ignore error return codes because if it was already disabled via
11424          * hardware settings this will fail
11425          */
11426         if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
11427                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
11428                 i40e_aq_stop_lldp(hw, true, NULL);
11429         }
11430
11431         /* allow a platform config to override the HW addr */
11432         i40e_get_platform_mac_addr(pdev, pf);
11433
11434         if (!is_valid_ether_addr(hw->mac.addr)) {
11435                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
11436                 err = -EIO;
11437                 goto err_mac_addr;
11438         }
11439         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
11440         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
11441         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
11442         if (is_valid_ether_addr(hw->mac.port_addr))
11443                 pf->hw_features |= I40E_HW_PORT_ID_VALID;
11444
11445         pci_set_drvdata(pdev, pf);
11446         pci_save_state(pdev);
11447 #ifdef CONFIG_I40E_DCB
11448         err = i40e_init_pf_dcb(pf);
11449         if (err) {
11450                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
11451                 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
11452                 /* Continue without DCB enabled */
11453         }
11454 #endif /* CONFIG_I40E_DCB */
11455
11456         /* set up periodic task facility */
11457         setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf);
11458         pf->service_timer_period = HZ;
11459
11460         INIT_WORK(&pf->service_task, i40e_service_task);
11461         clear_bit(__I40E_SERVICE_SCHED, pf->state);
11462
11463         /* NVM bit on means WoL disabled for the port */
11464         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
11465         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
11466                 pf->wol_en = false;
11467         else
11468                 pf->wol_en = true;
11469         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
11470
11471         /* set up the main switch operations */
11472         i40e_determine_queue_usage(pf);
11473         err = i40e_init_interrupt_scheme(pf);
11474         if (err)
11475                 goto err_switch_setup;
11476
11477         /* The number of VSIs reported by the FW is the minimum guaranteed
11478          * to us; HW supports far more and we share the remaining pool with
11479          * the other PFs. We allocate space for more than the guarantee with
11480          * the understanding that we might not get them all later.
11481          */
11482         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
11483                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
11484         else
11485                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
11486
11487         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
11488         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
11489                           GFP_KERNEL);
11490         if (!pf->vsi) {
11491                 err = -ENOMEM;
11492                 goto err_switch_setup;
11493         }
11494
11495 #ifdef CONFIG_PCI_IOV
11496         /* prep for VF support */
11497         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11498             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11499             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
11500                 if (pci_num_vf(pdev))
11501                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
11502         }
11503 #endif
11504         err = i40e_setup_pf_switch(pf, false);
11505         if (err) {
11506                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
11507                 goto err_vsis;
11508         }
11509
11510         /* Make sure flow control is set according to current settings */
11511         err = i40e_set_fc(hw, &set_fc_aq_fail, true);
11512         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET)
11513                 dev_dbg(&pf->pdev->dev,
11514                         "Set fc with err %s aq_err %s on get_phy_cap\n",
11515                         i40e_stat_str(hw, err),
11516                         i40e_aq_str(hw, hw->aq.asq_last_status));
11517         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET)
11518                 dev_dbg(&pf->pdev->dev,
11519                         "Set fc with err %s aq_err %s on set_phy_config\n",
11520                         i40e_stat_str(hw, err),
11521                         i40e_aq_str(hw, hw->aq.asq_last_status));
11522         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE)
11523                 dev_dbg(&pf->pdev->dev,
11524                         "Set fc with err %s aq_err %s on get_link_info\n",
11525                         i40e_stat_str(hw, err),
11526                         i40e_aq_str(hw, hw->aq.asq_last_status));
11527
11528         /* if FDIR VSI was set up, start it now */
11529         for (i = 0; i < pf->num_alloc_vsi; i++) {
11530                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
11531                         i40e_vsi_open(pf->vsi[i]);
11532                         break;
11533                 }
11534         }
11535
11536         /* The driver only wants link up/down and module qualification
11537          * reports from firmware.  Note the negative logic.
11538          */
11539         err = i40e_aq_set_phy_int_mask(&pf->hw,
11540                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
11541                                          I40E_AQ_EVENT_MEDIA_NA |
11542                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
11543         if (err)
11544                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
11545                          i40e_stat_str(&pf->hw, err),
11546                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11547
11548         /* Reconfigure hardware for allowing smaller MSS in the case
11549          * of TSO, so that we avoid the MDD being fired and causing
11550          * a reset in the case of small MSS+TSO.
11551          */
11552         val = rd32(hw, I40E_REG_MSS);
11553         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11554                 val &= ~I40E_REG_MSS_MIN_MASK;
11555                 val |= I40E_64BYTE_MSS;
11556                 wr32(hw, I40E_REG_MSS, val);
11557         }
11558
11559         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
11560                 msleep(75);
11561                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
11562                 if (err)
11563                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
11564                                  i40e_stat_str(&pf->hw, err),
11565                                  i40e_aq_str(&pf->hw,
11566                                              pf->hw.aq.asq_last_status));
11567         }
11568         /* The main driver is (mostly) up and happy. We need to set this state
11569          * before setting up the misc vector or we get a race and the vector
11570          * ends up disabled forever.
11571          */
11572         clear_bit(__I40E_DOWN, pf->state);
11573
11574         /* In case of MSIX we are going to setup the misc vector right here
11575          * to handle admin queue events etc. In case of legacy and MSI
11576          * the misc functionality and queue processing is combined in
11577          * the same vector and that gets setup at open.
11578          */
11579         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11580                 err = i40e_setup_misc_vector(pf);
11581                 if (err) {
11582                         dev_info(&pdev->dev,
11583                                  "setup of misc vector failed: %d\n", err);
11584                         goto err_vsis;
11585                 }
11586         }
11587
11588 #ifdef CONFIG_PCI_IOV
11589         /* prep for VF support */
11590         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11591             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11592             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
11593                 /* disable link interrupts for VFs */
11594                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
11595                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
11596                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
11597                 i40e_flush(hw);
11598
11599                 if (pci_num_vf(pdev)) {
11600                         dev_info(&pdev->dev,
11601                                  "Active VFs found, allocating resources.\n");
11602                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
11603                         if (err)
11604                                 dev_info(&pdev->dev,
11605                                          "Error %d allocating resources for existing VFs\n",
11606                                          err);
11607                 }
11608         }
11609 #endif /* CONFIG_PCI_IOV */
11610
11611         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11612                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
11613                                                       pf->num_iwarp_msix,
11614                                                       I40E_IWARP_IRQ_PILE_ID);
11615                 if (pf->iwarp_base_vector < 0) {
11616                         dev_info(&pdev->dev,
11617                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
11618                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
11619                         pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11620                 }
11621         }
11622
11623         i40e_dbg_pf_init(pf);
11624
11625         /* tell the firmware that we're starting */
11626         i40e_send_version(pf);
11627
11628         /* since everything's happy, start the service_task timer */
11629         mod_timer(&pf->service_timer,
11630                   round_jiffies(jiffies + pf->service_timer_period));
11631
11632         /* add this PF to client device list and launch a client service task */
11633         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11634                 err = i40e_lan_add_device(pf);
11635                 if (err)
11636                         dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
11637                                  err);
11638         }
11639
11640 #define PCI_SPEED_SIZE 8
11641 #define PCI_WIDTH_SIZE 8
11642         /* Devices on the IOSF bus do not have this information
11643          * and will report PCI Gen 1 x 1 by default so don't bother
11644          * checking them.
11645          */
11646         if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
11647                 char speed[PCI_SPEED_SIZE] = "Unknown";
11648                 char width[PCI_WIDTH_SIZE] = "Unknown";
11649
11650                 /* Get the negotiated link width and speed from PCI config
11651                  * space
11652                  */
11653                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
11654                                           &link_status);
11655
11656                 i40e_set_pci_config_data(hw, link_status);
11657
11658                 switch (hw->bus.speed) {
11659                 case i40e_bus_speed_8000:
11660                         strncpy(speed, "8.0", PCI_SPEED_SIZE); break;
11661                 case i40e_bus_speed_5000:
11662                         strncpy(speed, "5.0", PCI_SPEED_SIZE); break;
11663                 case i40e_bus_speed_2500:
11664                         strncpy(speed, "2.5", PCI_SPEED_SIZE); break;
11665                 default:
11666                         break;
11667                 }
11668                 switch (hw->bus.width) {
11669                 case i40e_bus_width_pcie_x8:
11670                         strncpy(width, "8", PCI_WIDTH_SIZE); break;
11671                 case i40e_bus_width_pcie_x4:
11672                         strncpy(width, "4", PCI_WIDTH_SIZE); break;
11673                 case i40e_bus_width_pcie_x2:
11674                         strncpy(width, "2", PCI_WIDTH_SIZE); break;
11675                 case i40e_bus_width_pcie_x1:
11676                         strncpy(width, "1", PCI_WIDTH_SIZE); break;
11677                 default:
11678                         break;
11679                 }
11680
11681                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
11682                          speed, width);
11683
11684                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
11685                     hw->bus.speed < i40e_bus_speed_8000) {
11686                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
11687                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
11688                 }
11689         }
11690
11691         /* get the requested speeds from the fw */
11692         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
11693         if (err)
11694                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %s last_status =  %s\n",
11695                         i40e_stat_str(&pf->hw, err),
11696                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11697         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
11698
11699         /* get the supported phy types from the fw */
11700         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
11701         if (err)
11702                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %s last_status =  %s\n",
11703                         i40e_stat_str(&pf->hw, err),
11704                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11705
11706         /* Add a filter to drop all Flow control frames from any VSI from being
11707          * transmitted. By doing so we stop a malicious VF from sending out
11708          * PAUSE or PFC frames and potentially controlling traffic for other
11709          * PF/VF VSIs.
11710          * The FW can still send Flow control frames if enabled.
11711          */
11712         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
11713                                                        pf->main_vsi_seid);
11714
11715         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
11716                 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
11717                 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
11718         if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
11719                 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
11720         /* print a string summarizing features */
11721         i40e_print_features(pf);
11722
11723         return 0;
11724
11725         /* Unwind what we've done if something failed in the setup */
11726 err_vsis:
11727         set_bit(__I40E_DOWN, pf->state);
11728         i40e_clear_interrupt_scheme(pf);
11729         kfree(pf->vsi);
11730 err_switch_setup:
11731         i40e_reset_interrupt_capability(pf);
11732         del_timer_sync(&pf->service_timer);
11733 err_mac_addr:
11734 err_configure_lan_hmc:
11735         (void)i40e_shutdown_lan_hmc(hw);
11736 err_init_lan_hmc:
11737         kfree(pf->qp_pile);
11738 err_sw_init:
11739 err_adminq_setup:
11740 err_pf_reset:
11741         iounmap(hw->hw_addr);
11742 err_ioremap:
11743         kfree(pf);
11744 err_pf_alloc:
11745         pci_disable_pcie_error_reporting(pdev);
11746         pci_release_mem_regions(pdev);
11747 err_pci_reg:
11748 err_dma:
11749         pci_disable_device(pdev);
11750         return err;
11751 }
11752
11753 /**
11754  * i40e_remove - Device removal routine
11755  * @pdev: PCI device information struct
11756  *
11757  * i40e_remove is called by the PCI subsystem to alert the driver
11758  * that is should release a PCI device.  This could be caused by a
11759  * Hot-Plug event, or because the driver is going to be removed from
11760  * memory.
11761  **/
11762 static void i40e_remove(struct pci_dev *pdev)
11763 {
11764         struct i40e_pf *pf = pci_get_drvdata(pdev);
11765         struct i40e_hw *hw = &pf->hw;
11766         i40e_status ret_code;
11767         int i;
11768
11769         i40e_dbg_pf_exit(pf);
11770
11771         i40e_ptp_stop(pf);
11772
11773         /* Disable RSS in hw */
11774         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
11775         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
11776
11777         /* no more scheduling of any task */
11778         set_bit(__I40E_SUSPENDED, pf->state);
11779         set_bit(__I40E_DOWN, pf->state);
11780         if (pf->service_timer.data)
11781                 del_timer_sync(&pf->service_timer);
11782         if (pf->service_task.func)
11783                 cancel_work_sync(&pf->service_task);
11784
11785         /* Client close must be called explicitly here because the timer
11786          * has been stopped.
11787          */
11788         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
11789
11790         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
11791                 i40e_free_vfs(pf);
11792                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
11793         }
11794
11795         i40e_fdir_teardown(pf);
11796
11797         /* If there is a switch structure or any orphans, remove them.
11798          * This will leave only the PF's VSI remaining.
11799          */
11800         for (i = 0; i < I40E_MAX_VEB; i++) {
11801                 if (!pf->veb[i])
11802                         continue;
11803
11804                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
11805                     pf->veb[i]->uplink_seid == 0)
11806                         i40e_switch_branch_release(pf->veb[i]);
11807         }
11808
11809         /* Now we can shutdown the PF's VSI, just before we kill
11810          * adminq and hmc.
11811          */
11812         if (pf->vsi[pf->lan_vsi])
11813                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
11814
11815         /* remove attached clients */
11816         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11817                 ret_code = i40e_lan_del_device(pf);
11818                 if (ret_code)
11819                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
11820                                  ret_code);
11821         }
11822
11823         /* shutdown and destroy the HMC */
11824         if (hw->hmc.hmc_obj) {
11825                 ret_code = i40e_shutdown_lan_hmc(hw);
11826                 if (ret_code)
11827                         dev_warn(&pdev->dev,
11828                                  "Failed to destroy the HMC resources: %d\n",
11829                                  ret_code);
11830         }
11831
11832         /* shutdown the adminq */
11833         i40e_shutdown_adminq(hw);
11834
11835         /* destroy the locks only once, here */
11836         mutex_destroy(&hw->aq.arq_mutex);
11837         mutex_destroy(&hw->aq.asq_mutex);
11838
11839         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
11840         i40e_clear_interrupt_scheme(pf);
11841         for (i = 0; i < pf->num_alloc_vsi; i++) {
11842                 if (pf->vsi[i]) {
11843                         i40e_vsi_clear_rings(pf->vsi[i]);
11844                         i40e_vsi_clear(pf->vsi[i]);
11845                         pf->vsi[i] = NULL;
11846                 }
11847         }
11848
11849         for (i = 0; i < I40E_MAX_VEB; i++) {
11850                 kfree(pf->veb[i]);
11851                 pf->veb[i] = NULL;
11852         }
11853
11854         kfree(pf->qp_pile);
11855         kfree(pf->vsi);
11856
11857         iounmap(hw->hw_addr);
11858         kfree(pf);
11859         pci_release_mem_regions(pdev);
11860
11861         pci_disable_pcie_error_reporting(pdev);
11862         pci_disable_device(pdev);
11863 }
11864
11865 /**
11866  * i40e_pci_error_detected - warning that something funky happened in PCI land
11867  * @pdev: PCI device information struct
11868  *
11869  * Called to warn that something happened and the error handling steps
11870  * are in progress.  Allows the driver to quiesce things, be ready for
11871  * remediation.
11872  **/
11873 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
11874                                                 enum pci_channel_state error)
11875 {
11876         struct i40e_pf *pf = pci_get_drvdata(pdev);
11877
11878         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
11879
11880         if (!pf) {
11881                 dev_info(&pdev->dev,
11882                          "Cannot recover - error happened during device probe\n");
11883                 return PCI_ERS_RESULT_DISCONNECT;
11884         }
11885
11886         /* shutdown all operations */
11887         if (!test_bit(__I40E_SUSPENDED, pf->state))
11888                 i40e_prep_for_reset(pf, false);
11889
11890         /* Request a slot reset */
11891         return PCI_ERS_RESULT_NEED_RESET;
11892 }
11893
11894 /**
11895  * i40e_pci_error_slot_reset - a PCI slot reset just happened
11896  * @pdev: PCI device information struct
11897  *
11898  * Called to find if the driver can work with the device now that
11899  * the pci slot has been reset.  If a basic connection seems good
11900  * (registers are readable and have sane content) then return a
11901  * happy little PCI_ERS_RESULT_xxx.
11902  **/
11903 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
11904 {
11905         struct i40e_pf *pf = pci_get_drvdata(pdev);
11906         pci_ers_result_t result;
11907         int err;
11908         u32 reg;
11909
11910         dev_dbg(&pdev->dev, "%s\n", __func__);
11911         if (pci_enable_device_mem(pdev)) {
11912                 dev_info(&pdev->dev,
11913                          "Cannot re-enable PCI device after reset.\n");
11914                 result = PCI_ERS_RESULT_DISCONNECT;
11915         } else {
11916                 pci_set_master(pdev);
11917                 pci_restore_state(pdev);
11918                 pci_save_state(pdev);
11919                 pci_wake_from_d3(pdev, false);
11920
11921                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
11922                 if (reg == 0)
11923                         result = PCI_ERS_RESULT_RECOVERED;
11924                 else
11925                         result = PCI_ERS_RESULT_DISCONNECT;
11926         }
11927
11928         err = pci_cleanup_aer_uncorrect_error_status(pdev);
11929         if (err) {
11930                 dev_info(&pdev->dev,
11931                          "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
11932                          err);
11933                 /* non-fatal, continue */
11934         }
11935
11936         return result;
11937 }
11938
11939 /**
11940  * i40e_pci_error_resume - restart operations after PCI error recovery
11941  * @pdev: PCI device information struct
11942  *
11943  * Called to allow the driver to bring things back up after PCI error
11944  * and/or reset recovery has finished.
11945  **/
11946 static void i40e_pci_error_resume(struct pci_dev *pdev)
11947 {
11948         struct i40e_pf *pf = pci_get_drvdata(pdev);
11949
11950         dev_dbg(&pdev->dev, "%s\n", __func__);
11951         if (test_bit(__I40E_SUSPENDED, pf->state))
11952                 return;
11953
11954         i40e_handle_reset_warning(pf, false);
11955 }
11956
11957 /**
11958  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
11959  * using the mac_address_write admin q function
11960  * @pf: pointer to i40e_pf struct
11961  **/
11962 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
11963 {
11964         struct i40e_hw *hw = &pf->hw;
11965         i40e_status ret;
11966         u8 mac_addr[6];
11967         u16 flags = 0;
11968
11969         /* Get current MAC address in case it's an LAA */
11970         if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
11971                 ether_addr_copy(mac_addr,
11972                                 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
11973         } else {
11974                 dev_err(&pf->pdev->dev,
11975                         "Failed to retrieve MAC address; using default\n");
11976                 ether_addr_copy(mac_addr, hw->mac.addr);
11977         }
11978
11979         /* The FW expects the mac address write cmd to first be called with
11980          * one of these flags before calling it again with the multicast
11981          * enable flags.
11982          */
11983         flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
11984
11985         if (hw->func_caps.flex10_enable && hw->partition_id != 1)
11986                 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
11987
11988         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
11989         if (ret) {
11990                 dev_err(&pf->pdev->dev,
11991                         "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
11992                 return;
11993         }
11994
11995         flags = I40E_AQC_MC_MAG_EN
11996                         | I40E_AQC_WOL_PRESERVE_ON_PFR
11997                         | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
11998         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
11999         if (ret)
12000                 dev_err(&pf->pdev->dev,
12001                         "Failed to enable Multicast Magic Packet wake up\n");
12002 }
12003
12004 /**
12005  * i40e_shutdown - PCI callback for shutting down
12006  * @pdev: PCI device information struct
12007  **/
12008 static void i40e_shutdown(struct pci_dev *pdev)
12009 {
12010         struct i40e_pf *pf = pci_get_drvdata(pdev);
12011         struct i40e_hw *hw = &pf->hw;
12012
12013         set_bit(__I40E_SUSPENDED, pf->state);
12014         set_bit(__I40E_DOWN, pf->state);
12015         rtnl_lock();
12016         i40e_prep_for_reset(pf, true);
12017         rtnl_unlock();
12018
12019         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
12020         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
12021
12022         del_timer_sync(&pf->service_timer);
12023         cancel_work_sync(&pf->service_task);
12024         i40e_fdir_teardown(pf);
12025
12026         /* Client close must be called explicitly here because the timer
12027          * has been stopped.
12028          */
12029         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
12030
12031         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
12032                 i40e_enable_mc_magic_wake(pf);
12033
12034         i40e_prep_for_reset(pf, false);
12035
12036         wr32(hw, I40E_PFPM_APM,
12037              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
12038         wr32(hw, I40E_PFPM_WUFC,
12039              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
12040
12041         i40e_clear_interrupt_scheme(pf);
12042
12043         if (system_state == SYSTEM_POWER_OFF) {
12044                 pci_wake_from_d3(pdev, pf->wol_en);
12045                 pci_set_power_state(pdev, PCI_D3hot);
12046         }
12047 }
12048
12049 #ifdef CONFIG_PM
12050 /**
12051  * i40e_suspend - PCI callback for moving to D3
12052  * @pdev: PCI device information struct
12053  **/
12054 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
12055 {
12056         struct i40e_pf *pf = pci_get_drvdata(pdev);
12057         struct i40e_hw *hw = &pf->hw;
12058         int retval = 0;
12059
12060         set_bit(__I40E_SUSPENDED, pf->state);
12061         set_bit(__I40E_DOWN, pf->state);
12062
12063         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
12064                 i40e_enable_mc_magic_wake(pf);
12065
12066         i40e_prep_for_reset(pf, false);
12067
12068         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
12069         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
12070
12071         i40e_stop_misc_vector(pf);
12072         if (pf->msix_entries) {
12073                 synchronize_irq(pf->msix_entries[0].vector);
12074                 free_irq(pf->msix_entries[0].vector, pf);
12075         }
12076         retval = pci_save_state(pdev);
12077         if (retval)
12078                 return retval;
12079
12080         pci_wake_from_d3(pdev, pf->wol_en);
12081         pci_set_power_state(pdev, PCI_D3hot);
12082
12083         return retval;
12084 }
12085
12086 /**
12087  * i40e_resume - PCI callback for waking up from D3
12088  * @pdev: PCI device information struct
12089  **/
12090 static int i40e_resume(struct pci_dev *pdev)
12091 {
12092         struct i40e_pf *pf = pci_get_drvdata(pdev);
12093         u32 err;
12094
12095         pci_set_power_state(pdev, PCI_D0);
12096         pci_restore_state(pdev);
12097         /* pci_restore_state() clears dev->state_saves, so
12098          * call pci_save_state() again to restore it.
12099          */
12100         pci_save_state(pdev);
12101
12102         err = pci_enable_device_mem(pdev);
12103         if (err) {
12104                 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
12105                 return err;
12106         }
12107         pci_set_master(pdev);
12108
12109         /* no wakeup events while running */
12110         pci_wake_from_d3(pdev, false);
12111
12112         /* handling the reset will rebuild the device state */
12113         if (test_and_clear_bit(__I40E_SUSPENDED, pf->state)) {
12114                 clear_bit(__I40E_DOWN, pf->state);
12115                 if (pf->msix_entries) {
12116                         err = request_irq(pf->msix_entries[0].vector,
12117                                           i40e_intr, 0, pf->int_name, pf);
12118                         if (err) {
12119                                 dev_err(&pf->pdev->dev,
12120                                         "request_irq for %s failed: %d\n",
12121                                         pf->int_name, err);
12122                         }
12123                 }
12124                 i40e_reset_and_rebuild(pf, false, false);
12125         }
12126
12127         return 0;
12128 }
12129
12130 #endif
12131 static const struct pci_error_handlers i40e_err_handler = {
12132         .error_detected = i40e_pci_error_detected,
12133         .slot_reset = i40e_pci_error_slot_reset,
12134         .resume = i40e_pci_error_resume,
12135 };
12136
12137 static struct pci_driver i40e_driver = {
12138         .name     = i40e_driver_name,
12139         .id_table = i40e_pci_tbl,
12140         .probe    = i40e_probe,
12141         .remove   = i40e_remove,
12142 #ifdef CONFIG_PM
12143         .suspend  = i40e_suspend,
12144         .resume   = i40e_resume,
12145 #endif
12146         .shutdown = i40e_shutdown,
12147         .err_handler = &i40e_err_handler,
12148         .sriov_configure = i40e_pci_sriov_configure,
12149 };
12150
12151 /**
12152  * i40e_init_module - Driver registration routine
12153  *
12154  * i40e_init_module is the first routine called when the driver is
12155  * loaded. All it does is register with the PCI subsystem.
12156  **/
12157 static int __init i40e_init_module(void)
12158 {
12159         pr_info("%s: %s - version %s\n", i40e_driver_name,
12160                 i40e_driver_string, i40e_driver_version_str);
12161         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
12162
12163         /* There is no need to throttle the number of active tasks because
12164          * each device limits its own task using a state bit for scheduling
12165          * the service task, and the device tasks do not interfere with each
12166          * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
12167          * since we need to be able to guarantee forward progress even under
12168          * memory pressure.
12169          */
12170         i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
12171         if (!i40e_wq) {
12172                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
12173                 return -ENOMEM;
12174         }
12175
12176         i40e_dbg_init();
12177         return pci_register_driver(&i40e_driver);
12178 }
12179 module_init(i40e_init_module);
12180
12181 /**
12182  * i40e_exit_module - Driver exit cleanup routine
12183  *
12184  * i40e_exit_module is called just before the driver is removed
12185  * from memory.
12186  **/
12187 static void __exit i40e_exit_module(void)
12188 {
12189         pci_unregister_driver(&i40e_driver);
12190         destroy_workqueue(i40e_wq);
12191         i40e_dbg_exit();
12192 }
12193 module_exit(i40e_exit_module);