drivers/net/ethernet/intel/i40e/i40e_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2017 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 #include <linux/etherdevice.h>
  28 #include <linux/of_net.h>
  29 #include <linux/pci.h>
  30 #include <linux/bpf.h>
  31
  32 /* Local includes */
  33 #include "i40e.h"
  34 #include "i40e_diag.h"
  35 #include <net/udp_tunnel.h>
  36 /* All i40e tracepoints are defined by the include below, which
  37  * must be included exactly once across the whole kernel with
  38  * CREATE_TRACE_POINTS defined
  39  */
  40 #define CREATE_TRACE_POINTS
  41 #include "i40e_trace.h"
  42
  43 const char i40e_driver_name[] = "i40e";
  44 static const char i40e_driver_string[] =
  45                         "Intel(R) Ethernet Connection XL710 Network Driver";
  46
  47 #define DRV_KERN "-k"
  48
  49 #define DRV_VERSION_MAJOR 2
  50 #define DRV_VERSION_MINOR 1
  51 #define DRV_VERSION_BUILD 14
  52 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  53              __stringify(DRV_VERSION_MINOR) "." \
  54              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  55 const char i40e_driver_version_str[] = DRV_VERSION;
  56 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation.";
  57
  58 /* a bit of forward declarations */
  59 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  60 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
  61 static int i40e_add_vsi(struct i40e_vsi *vsi);
  62 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  63 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  64 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  65 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  66 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  67 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired);
  68 static int i40e_reset(struct i40e_pf *pf);
  69 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  70 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  71 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  72 static int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
  73                                      struct i40e_cloud_filter *filter,
  74                                      bool add);
  75 static int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
  76                                              struct i40e_cloud_filter *filter,
  77                                              bool add);
  78 static int i40e_get_capabilities(struct i40e_pf *pf,
  79                                  enum i40e_admin_queue_opc list_type);
  80
  81
  82 /* i40e_pci_tbl - PCI Device ID Table
  83  *
  84  * Last entry must be all 0s
  85  *
  86  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  87  *   Class, Class Mask, private data (not used) }
  88  */
  89 static const struct pci_device_id i40e_pci_tbl[] = {
  90         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  91         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  92         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  93         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  94         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  95         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  96         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  97         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  98         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
  99         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
 100         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
 101         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
 102         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
 103         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
 104         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
 105         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
 106         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
 107         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
 108         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
 109         /* required last entry */
 110         {0, }
 111 };
 112 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
 113
 114 #define I40E_MAX_VF_COUNT 128
 115 static int debug = -1;
 116 module_param(debug, uint, 0);
 117 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
 118
 119 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 120 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
 121 MODULE_LICENSE("GPL");
 122 MODULE_VERSION(DRV_VERSION);
 123
 124 static struct workqueue_struct *i40e_wq;
 125
 126 /**
 127  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
 128  * @hw:   pointer to the HW structure
 129  * @mem:  ptr to mem struct to fill out
 130  * @size: size of memory requested
 131  * @alignment: what to align the allocation to
 132  **/
 133 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 134                             u64 size, u32 alignment)
 135 {
 136         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 137
 138         mem->size = ALIGN(size, alignment);
 139         mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
 140                                       &mem->pa, GFP_KERNEL);
 141         if (!mem->va)
 142                 return -ENOMEM;
 143
 144         return 0;
 145 }
 146
 147 /**
 148  * i40e_free_dma_mem_d - OS specific memory free for shared code
 149  * @hw:   pointer to the HW structure
 150  * @mem:  ptr to mem struct to free
 151  **/
 152 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 153 {
 154         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 155
 156         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 157         mem->va = NULL;
 158         mem->pa = 0;
 159         mem->size = 0;
 160
 161         return 0;
 162 }
 163
 164 /**
 165  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 166  * @hw:   pointer to the HW structure
 167  * @mem:  ptr to mem struct to fill out
 168  * @size: size of memory requested
 169  **/
 170 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 171                              u32 size)
 172 {
 173         mem->size = size;
 174         mem->va = kzalloc(size, GFP_KERNEL);
 175
 176         if (!mem->va)
 177                 return -ENOMEM;
 178
 179         return 0;
 180 }
 181
 182 /**
 183  * i40e_free_virt_mem_d - OS specific memory free for shared code
 184  * @hw:   pointer to the HW structure
 185  * @mem:  ptr to mem struct to free
 186  **/
 187 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 188 {
 189         /* it's ok to kfree a NULL pointer */
 190         kfree(mem->va);
 191         mem->va = NULL;
 192         mem->size = 0;
 193
 194         return 0;
 195 }
 196
 197 /**
 198  * i40e_get_lump - find a lump of free generic resource
 199  * @pf: board private structure
 200  * @pile: the pile of resource to search
 201  * @needed: the number of items needed
 202  * @id: an owner id to stick on the items assigned
 203  *
 204  * Returns the base item index of the lump, or negative for error
 205  *
 206  * The search_hint trick and lack of advanced fit-finding only work
 207  * because we're highly likely to have all the same size lump requests.
 208  * Linear search time and any fragmentation should be minimal.
 209  **/
 210 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 211                          u16 needed, u16 id)
 212 {
 213         int ret = -ENOMEM;
 214         int i, j;
 215
 216         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 217                 dev_info(&pf->pdev->dev,
 218                          "param err: pile=%p needed=%d id=0x%04x\n",
 219                          pile, needed, id);
 220                 return -EINVAL;
 221         }
 222
 223         /* start the linear search with an imperfect hint */
 224         i = pile->search_hint;
 225         while (i < pile->num_entries) {
 226                 /* skip already allocated entries */
 227                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 228                         i++;
 229                         continue;
 230                 }
 231
 232                 /* do we have enough in this lump? */
 233                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 234                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 235                                 break;
 236                 }
 237
 238                 if (j == needed) {
 239                         /* there was enough, so assign it to the requestor */
 240                         for (j = 0; j < needed; j++)
 241                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 242                         ret = i;
 243                         pile->search_hint = i + j;
 244                         break;
 245                 }
 246
 247                 /* not enough, so skip over it and continue looking */
 248                 i += j;
 249         }
 250
 251         return ret;
 252 }
 253
 254 /**
 255  * i40e_put_lump - return a lump of generic resource
 256  * @pile: the pile of resource to search
 257  * @index: the base item index
 258  * @id: the owner id of the items assigned
 259  *
 260  * Returns the count of items in the lump
 261  **/
 262 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 263 {
 264         int valid_id = (id | I40E_PILE_VALID_BIT);
 265         int count = 0;
 266         int i;
 267
 268         if (!pile || index >= pile->num_entries)
 269                 return -EINVAL;
 270
 271         for (i = index;
 272              i < pile->num_entries && pile->list[i] == valid_id;
 273              i++) {
 274                 pile->list[i] = 0;
 275                 count++;
 276         }
 277
 278         if (count && index < pile->search_hint)
 279                 pile->search_hint = index;
 280
 281         return count;
 282 }
 283
 284 /**
 285  * i40e_find_vsi_from_id - searches for the vsi with the given id
 286  * @pf - the pf structure to search for the vsi
 287  * @id - id of the vsi it is searching for
 288  **/
 289 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 290 {
 291         int i;
 292
 293         for (i = 0; i < pf->num_alloc_vsi; i++)
 294                 if (pf->vsi[i] && (pf->vsi[i]->id == id))
 295                         return pf->vsi[i];
 296
 297         return NULL;
 298 }
 299
 300 /**
 301  * i40e_service_event_schedule - Schedule the service task to wake up
 302  * @pf: board private structure
 303  *
 304  * If not already scheduled, this puts the task into the work queue
 305  **/
 306 void i40e_service_event_schedule(struct i40e_pf *pf)
 307 {
 308         if (!test_bit(__I40E_DOWN, pf->state) &&
 309             !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
 310                 queue_work(i40e_wq, &pf->service_task);
 311 }
 312
 313 /**
 314  * i40e_tx_timeout - Respond to a Tx Hang
 315  * @netdev: network interface device structure
 316  *
 317  * If any port has noticed a Tx timeout, it is likely that the whole
 318  * device is munged, not just the one netdev port, so go for the full
 319  * reset.
 320  **/
 321 static void i40e_tx_timeout(struct net_device *netdev)
 322 {
 323         struct i40e_netdev_priv *np = netdev_priv(netdev);
 324         struct i40e_vsi *vsi = np->vsi;
 325         struct i40e_pf *pf = vsi->back;
 326         struct i40e_ring *tx_ring = NULL;
 327         unsigned int i, hung_queue = 0;
 328         u32 head, val;
 329
 330         pf->tx_timeout_count++;
 331
 332         /* find the stopped queue the same way the stack does */
 333         for (i = 0; i < netdev->num_tx_queues; i++) {
 334                 struct netdev_queue *q;
 335                 unsigned long trans_start;
 336
 337                 q = netdev_get_tx_queue(netdev, i);
 338                 trans_start = q->trans_start;
 339                 if (netif_xmit_stopped(q) &&
 340                     time_after(jiffies,
 341                                (trans_start + netdev->watchdog_timeo))) {
 342                         hung_queue = i;
 343                         break;
 344                 }
 345         }
 346
 347         if (i == netdev->num_tx_queues) {
 348                 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n");
 349         } else {
 350                 /* now that we have an index, find the tx_ring struct */
 351                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 352                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
 353                                 if (hung_queue ==
 354                                     vsi->tx_rings[i]->queue_index) {
 355                                         tx_ring = vsi->tx_rings[i];
 356                                         break;
 357                                 }
 358                         }
 359                 }
 360         }
 361
 362         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 363                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
 364         else if (time_before(jiffies,
 365                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
 366                 return;   /* don't do any new action before the next timeout */
 367
 368         if (tx_ring) {
 369                 head = i40e_get_head(tx_ring);
 370                 /* Read interrupt register */
 371                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
 372                         val = rd32(&pf->hw,
 373                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 374                                                 tx_ring->vsi->base_vector - 1));
 375                 else
 376                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
 377
 378                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
 379                             vsi->seid, hung_queue, tx_ring->next_to_clean,
 380                             head, tx_ring->next_to_use,
 381                             readl(tx_ring->tail), val);
 382         }
 383
 384         pf->tx_timeout_last_recovery = jiffies;
 385         netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n",
 386                     pf->tx_timeout_recovery_level, hung_queue);
 387
 388         switch (pf->tx_timeout_recovery_level) {
 389         case 1:
 390                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
 391                 break;
 392         case 2:
 393                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
 394                 break;
 395         case 3:
 396                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
 397                 break;
 398         default:
 399                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 400                 break;
 401         }
 402
 403         i40e_service_event_schedule(pf);
 404         pf->tx_timeout_recovery_level++;
 405 }
 406
 407 /**
 408  * i40e_get_vsi_stats_struct - Get System Network Statistics
 409  * @vsi: the VSI we care about
 410  *
 411  * Returns the address of the device statistics structure.
 412  * The statistics are actually updated from the service task.
 413  **/
 414 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 415 {
 416         return &vsi->net_stats;
 417 }
 418
 419 /**
 420  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
 421  * @ring: Tx ring to get statistics from
 422  * @stats: statistics entry to be updated
 423  **/
 424 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
 425                                             struct rtnl_link_stats64 *stats)
 426 {
 427         u64 bytes, packets;
 428         unsigned int start;
 429
 430         do {
 431                 start = u64_stats_fetch_begin_irq(&ring->syncp);
 432                 packets = ring->stats.packets;
 433                 bytes   = ring->stats.bytes;
 434         } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 435
 436         stats->tx_packets += packets;
 437         stats->tx_bytes   += bytes;
 438 }
 439
 440 /**
 441  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 442  * @netdev: network interface device structure
 443  *
 444  * Returns the address of the device statistics structure.
 445  * The statistics are actually updated from the service task.
 446  **/
 447 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
 448                                   struct rtnl_link_stats64 *stats)
 449 {
 450         struct i40e_netdev_priv *np = netdev_priv(netdev);
 451         struct i40e_ring *tx_ring, *rx_ring;
 452         struct i40e_vsi *vsi = np->vsi;
 453         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 454         int i;
 455
 456         if (test_bit(__I40E_VSI_DOWN, vsi->state))
 457                 return;
 458
 459         if (!vsi->tx_rings)
 460                 return;
 461
 462         rcu_read_lock();
 463         for (i = 0; i < vsi->num_queue_pairs; i++) {
 464                 u64 bytes, packets;
 465                 unsigned int start;
 466
 467                 tx_ring = READ_ONCE(vsi->tx_rings[i]);
 468                 if (!tx_ring)
 469                         continue;
 470                 i40e_get_netdev_stats_struct_tx(tx_ring, stats);
 471
 472                 rx_ring = &tx_ring[1];
 473
 474                 do {
 475                         start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
 476                         packets = rx_ring->stats.packets;
 477                         bytes   = rx_ring->stats.bytes;
 478                 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
 479
 480                 stats->rx_packets += packets;
 481                 stats->rx_bytes   += bytes;
 482
 483                 if (i40e_enabled_xdp_vsi(vsi))
 484                         i40e_get_netdev_stats_struct_tx(&rx_ring[1], stats);
 485         }
 486         rcu_read_unlock();
 487
 488         /* following stats updated by i40e_watchdog_subtask() */
 489         stats->multicast        = vsi_stats->multicast;
 490         stats->tx_errors        = vsi_stats->tx_errors;
 491         stats->tx_dropped       = vsi_stats->tx_dropped;
 492         stats->rx_errors        = vsi_stats->rx_errors;
 493         stats->rx_dropped       = vsi_stats->rx_dropped;
 494         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 495         stats->rx_length_errors = vsi_stats->rx_length_errors;
 496 }
 497
 498 /**
 499  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 500  * @vsi: the VSI to have its stats reset
 501  **/
 502 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 503 {
 504         struct rtnl_link_stats64 *ns;
 505         int i;
 506
 507         if (!vsi)
 508                 return;
 509
 510         ns = i40e_get_vsi_stats_struct(vsi);
 511         memset(ns, 0, sizeof(*ns));
 512         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 513         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 514         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 515         if (vsi->rx_rings && vsi->rx_rings[0]) {
 516                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 517                         memset(&vsi->rx_rings[i]->stats, 0,
 518                                sizeof(vsi->rx_rings[i]->stats));
 519                         memset(&vsi->rx_rings[i]->rx_stats, 0,
 520                                sizeof(vsi->rx_rings[i]->rx_stats));
 521                         memset(&vsi->tx_rings[i]->stats, 0,
 522                                sizeof(vsi->tx_rings[i]->stats));
 523                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 524                                sizeof(vsi->tx_rings[i]->tx_stats));
 525                 }
 526         }
 527         vsi->stat_offsets_loaded = false;
 528 }
 529
 530 /**
 531  * i40e_pf_reset_stats - Reset all of the stats for the given PF
 532  * @pf: the PF to be reset
 533  **/
 534 void i40e_pf_reset_stats(struct i40e_pf *pf)
 535 {
 536         int i;
 537
 538         memset(&pf->stats, 0, sizeof(pf->stats));
 539         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 540         pf->stat_offsets_loaded = false;
 541
 542         for (i = 0; i < I40E_MAX_VEB; i++) {
 543                 if (pf->veb[i]) {
 544                         memset(&pf->veb[i]->stats, 0,
 545                                sizeof(pf->veb[i]->stats));
 546                         memset(&pf->veb[i]->stats_offsets, 0,
 547                                sizeof(pf->veb[i]->stats_offsets));
 548                         pf->veb[i]->stat_offsets_loaded = false;
 549                 }
 550         }
 551         pf->hw_csum_rx_error = 0;
 552 }
 553
 554 /**
 555  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 556  * @hw: ptr to the hardware info
 557  * @hireg: the high 32 bit reg to read
 558  * @loreg: the low 32 bit reg to read
 559  * @offset_loaded: has the initial offset been loaded yet
 560  * @offset: ptr to current offset value
 561  * @stat: ptr to the stat
 562  *
 563  * Since the device stats are not reset at PFReset, they likely will not
 564  * be zeroed when the driver starts.  We'll save the first values read
 565  * and use them as offsets to be subtracted from the raw values in order
 566  * to report stats that count from zero.  In the process, we also manage
 567  * the potential roll-over.
 568  **/
 569 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 570                                bool offset_loaded, u64 *offset, u64 *stat)
 571 {
 572         u64 new_data;
 573
 574         if (hw->device_id == I40E_DEV_ID_QEMU) {
 575                 new_data = rd32(hw, loreg);
 576                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 577         } else {
 578                 new_data = rd64(hw, loreg);
 579         }
 580         if (!offset_loaded)
 581                 *offset = new_data;
 582         if (likely(new_data >= *offset))
 583                 *stat = new_data - *offset;
 584         else
 585                 *stat = (new_data + BIT_ULL(48)) - *offset;
 586         *stat &= 0xFFFFFFFFFFFFULL;
 587 }
 588
 589 /**
 590  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 591  * @hw: ptr to the hardware info
 592  * @reg: the hw reg to read
 593  * @offset_loaded: has the initial offset been loaded yet
 594  * @offset: ptr to current offset value
 595  * @stat: ptr to the stat
 596  **/
 597 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 598                                bool offset_loaded, u64 *offset, u64 *stat)
 599 {
 600         u32 new_data;
 601
 602         new_data = rd32(hw, reg);
 603         if (!offset_loaded)
 604                 *offset = new_data;
 605         if (likely(new_data >= *offset))
 606                 *stat = (u32)(new_data - *offset);
 607         else
 608                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
 609 }
 610
 611 /**
 612  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
 613  * @hw: ptr to the hardware info
 614  * @reg: the hw reg to read and clear
 615  * @stat: ptr to the stat
 616  **/
 617 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
 618 {
 619         u32 new_data = rd32(hw, reg);
 620
 621         wr32(hw, reg, 1); /* must write a nonzero value to clear register */
 622         *stat += new_data;
 623 }
 624
 625 /**
 626  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 627  * @vsi: the VSI to be updated
 628  **/
 629 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 630 {
 631         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 632         struct i40e_pf *pf = vsi->back;
 633         struct i40e_hw *hw = &pf->hw;
 634         struct i40e_eth_stats *oes;
 635         struct i40e_eth_stats *es;     /* device's eth stats */
 636
 637         es = &vsi->eth_stats;
 638         oes = &vsi->eth_stats_offsets;
 639
 640         /* Gather up the stats that the hw collects */
 641         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 642                            vsi->stat_offsets_loaded,
 643                            &oes->tx_errors, &es->tx_errors);
 644         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 645                            vsi->stat_offsets_loaded,
 646                            &oes->rx_discards, &es->rx_discards);
 647         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 648                            vsi->stat_offsets_loaded,
 649                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 650         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 651                            vsi->stat_offsets_loaded,
 652                            &oes->tx_errors, &es->tx_errors);
 653
 654         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 655                            I40E_GLV_GORCL(stat_idx),
 656                            vsi->stat_offsets_loaded,
 657                            &oes->rx_bytes, &es->rx_bytes);
 658         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 659                            I40E_GLV_UPRCL(stat_idx),
 660                            vsi->stat_offsets_loaded,
 661                            &oes->rx_unicast, &es->rx_unicast);
 662         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 663                            I40E_GLV_MPRCL(stat_idx),
 664                            vsi->stat_offsets_loaded,
 665                            &oes->rx_multicast, &es->rx_multicast);
 666         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 667                            I40E_GLV_BPRCL(stat_idx),
 668                            vsi->stat_offsets_loaded,
 669                            &oes->rx_broadcast, &es->rx_broadcast);
 670
 671         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 672                            I40E_GLV_GOTCL(stat_idx),
 673                            vsi->stat_offsets_loaded,
 674                            &oes->tx_bytes, &es->tx_bytes);
 675         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 676                            I40E_GLV_UPTCL(stat_idx),
 677                            vsi->stat_offsets_loaded,
 678                            &oes->tx_unicast, &es->tx_unicast);
 679         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 680                            I40E_GLV_MPTCL(stat_idx),
 681                            vsi->stat_offsets_loaded,
 682                            &oes->tx_multicast, &es->tx_multicast);
 683         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 684                            I40E_GLV_BPTCL(stat_idx),
 685                            vsi->stat_offsets_loaded,
 686                            &oes->tx_broadcast, &es->tx_broadcast);
 687         vsi->stat_offsets_loaded = true;
 688 }
 689
 690 /**
 691  * i40e_update_veb_stats - Update Switch component statistics
 692  * @veb: the VEB being updated
 693  **/
 694 static void i40e_update_veb_stats(struct i40e_veb *veb)
 695 {
 696         struct i40e_pf *pf = veb->pf;
 697         struct i40e_hw *hw = &pf->hw;
 698         struct i40e_eth_stats *oes;
 699         struct i40e_eth_stats *es;     /* device's eth stats */
 700         struct i40e_veb_tc_stats *veb_oes;
 701         struct i40e_veb_tc_stats *veb_es;
 702         int i, idx = 0;
 703
 704         idx = veb->stats_idx;
 705         es = &veb->stats;
 706         oes = &veb->stats_offsets;
 707         veb_es = &veb->tc_stats;
 708         veb_oes = &veb->tc_stats_offsets;
 709
 710         /* Gather up the stats that the hw collects */
 711         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 712                            veb->stat_offsets_loaded,
 713                            &oes->tx_discards, &es->tx_discards);
 714         if (hw->revision_id > 0)
 715                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 716                                    veb->stat_offsets_loaded,
 717                                    &oes->rx_unknown_protocol,
 718                                    &es->rx_unknown_protocol);
 719         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 720                            veb->stat_offsets_loaded,
 721                            &oes->rx_bytes, &es->rx_bytes);
 722         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 723                            veb->stat_offsets_loaded,
 724                            &oes->rx_unicast, &es->rx_unicast);
 725         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 726                            veb->stat_offsets_loaded,
 727                            &oes->rx_multicast, &es->rx_multicast);
 728         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 729                            veb->stat_offsets_loaded,
 730                            &oes->rx_broadcast, &es->rx_broadcast);
 731
 732         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 733                            veb->stat_offsets_loaded,
 734                            &oes->tx_bytes, &es->tx_bytes);
 735         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 736                            veb->stat_offsets_loaded,
 737                            &oes->tx_unicast, &es->tx_unicast);
 738         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 739                            veb->stat_offsets_loaded,
 740                            &oes->tx_multicast, &es->tx_multicast);
 741         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 742                            veb->stat_offsets_loaded,
 743                            &oes->tx_broadcast, &es->tx_broadcast);
 744         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 745                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
 746                                    I40E_GLVEBTC_RPCL(i, idx),
 747                                    veb->stat_offsets_loaded,
 748                                    &veb_oes->tc_rx_packets[i],
 749                                    &veb_es->tc_rx_packets[i]);
 750                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
 751                                    I40E_GLVEBTC_RBCL(i, idx),
 752                                    veb->stat_offsets_loaded,
 753                                    &veb_oes->tc_rx_bytes[i],
 754                                    &veb_es->tc_rx_bytes[i]);
 755                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
 756                                    I40E_GLVEBTC_TPCL(i, idx),
 757                                    veb->stat_offsets_loaded,
 758                                    &veb_oes->tc_tx_packets[i],
 759                                    &veb_es->tc_tx_packets[i]);
 760                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
 761                                    I40E_GLVEBTC_TBCL(i, idx),
 762                                    veb->stat_offsets_loaded,
 763                                    &veb_oes->tc_tx_bytes[i],
 764                                    &veb_es->tc_tx_bytes[i]);
 765         }
 766         veb->stat_offsets_loaded = true;
 767 }
 768
 769 /**
 770  * i40e_update_vsi_stats - Update the vsi statistics counters.
 771  * @vsi: the VSI to be updated
 772  *
 773  * There are a few instances where we store the same stat in a
 774  * couple of different structs.  This is partly because we have
 775  * the netdev stats that need to be filled out, which is slightly
 776  * different from the "eth_stats" defined by the chip and used in
 777  * VF communications.  We sort it out here.
 778  **/
 779 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 780 {
 781         struct i40e_pf *pf = vsi->back;
 782         struct rtnl_link_stats64 *ons;
 783         struct rtnl_link_stats64 *ns;   /* netdev stats */
 784         struct i40e_eth_stats *oes;
 785         struct i40e_eth_stats *es;     /* device's eth stats */
 786         u32 tx_restart, tx_busy;
 787         struct i40e_ring *p;
 788         u32 rx_page, rx_buf;
 789         u64 bytes, packets;
 790         unsigned int start;
 791         u64 tx_linearize;
 792         u64 tx_force_wb;
 793         u64 rx_p, rx_b;
 794         u64 tx_p, tx_b;
 795         u16 q;
 796
 797         if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
 798             test_bit(__I40E_CONFIG_BUSY, pf->state))
 799                 return;
 800
 801         ns = i40e_get_vsi_stats_struct(vsi);
 802         ons = &vsi->net_stats_offsets;
 803         es = &vsi->eth_stats;
 804         oes = &vsi->eth_stats_offsets;
 805
 806         /* Gather up the netdev and vsi stats that the driver collects
 807          * on the fly during packet processing
 808          */
 809         rx_b = rx_p = 0;
 810         tx_b = tx_p = 0;
 811         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
 812         rx_page = 0;
 813         rx_buf = 0;
 814         rcu_read_lock();
 815         for (q = 0; q < vsi->num_queue_pairs; q++) {
 816                 /* locate Tx ring */
 817                 p = READ_ONCE(vsi->tx_rings[q]);
 818
 819                 do {
 820                         start = u64_stats_fetch_begin_irq(&p->syncp);
 821                         packets = p->stats.packets;
 822                         bytes = p->stats.bytes;
 823                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 824                 tx_b += bytes;
 825                 tx_p += packets;
 826                 tx_restart += p->tx_stats.restart_queue;
 827                 tx_busy += p->tx_stats.tx_busy;
 828                 tx_linearize += p->tx_stats.tx_linearize;
 829                 tx_force_wb += p->tx_stats.tx_force_wb;
 830
 831                 /* Rx queue is part of the same block as Tx queue */
 832                 p = &p[1];
 833                 do {
 834                         start = u64_stats_fetch_begin_irq(&p->syncp);
 835                         packets = p->stats.packets;
 836                         bytes = p->stats.bytes;
 837                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 838                 rx_b += bytes;
 839                 rx_p += packets;
 840                 rx_buf += p->rx_stats.alloc_buff_failed;
 841                 rx_page += p->rx_stats.alloc_page_failed;
 842         }
 843         rcu_read_unlock();
 844         vsi->tx_restart = tx_restart;
 845         vsi->tx_busy = tx_busy;
 846         vsi->tx_linearize = tx_linearize;
 847         vsi->tx_force_wb = tx_force_wb;
 848         vsi->rx_page_failed = rx_page;
 849         vsi->rx_buf_failed = rx_buf;
 850
 851         ns->rx_packets = rx_p;
 852         ns->rx_bytes = rx_b;
 853         ns->tx_packets = tx_p;
 854         ns->tx_bytes = tx_b;
 855
 856         /* update netdev stats from eth stats */
 857         i40e_update_eth_stats(vsi);
 858         ons->tx_errors = oes->tx_errors;
 859         ns->tx_errors = es->tx_errors;
 860         ons->multicast = oes->rx_multicast;
 861         ns->multicast = es->rx_multicast;
 862         ons->rx_dropped = oes->rx_discards;
 863         ns->rx_dropped = es->rx_discards;
 864         ons->tx_dropped = oes->tx_discards;
 865         ns->tx_dropped = es->tx_discards;
 866
 867         /* pull in a couple PF stats if this is the main vsi */
 868         if (vsi == pf->vsi[pf->lan_vsi]) {
 869                 ns->rx_crc_errors = pf->stats.crc_errors;
 870                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 871                 ns->rx_length_errors = pf->stats.rx_length_errors;
 872         }
 873 }
 874
 875 /**
 876  * i40e_update_pf_stats - Update the PF statistics counters.
 877  * @pf: the PF to be updated
 878  **/
 879 static void i40e_update_pf_stats(struct i40e_pf *pf)
 880 {
 881         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 882         struct i40e_hw_port_stats *nsd = &pf->stats;
 883         struct i40e_hw *hw = &pf->hw;
 884         u32 val;
 885         int i;
 886
 887         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 888                            I40E_GLPRT_GORCL(hw->port),
 889                            pf->stat_offsets_loaded,
 890                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 891         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 892                            I40E_GLPRT_GOTCL(hw->port),
 893                            pf->stat_offsets_loaded,
 894                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 895         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 896                            pf->stat_offsets_loaded,
 897                            &osd->eth.rx_discards,
 898                            &nsd->eth.rx_discards);
 899         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 900                            I40E_GLPRT_UPRCL(hw->port),
 901                            pf->stat_offsets_loaded,
 902                            &osd->eth.rx_unicast,
 903                            &nsd->eth.rx_unicast);
 904         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 905                            I40E_GLPRT_MPRCL(hw->port),
 906                            pf->stat_offsets_loaded,
 907                            &osd->eth.rx_multicast,
 908                            &nsd->eth.rx_multicast);
 909         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 910                            I40E_GLPRT_BPRCL(hw->port),
 911                            pf->stat_offsets_loaded,
 912                            &osd->eth.rx_broadcast,
 913                            &nsd->eth.rx_broadcast);
 914         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 915                            I40E_GLPRT_UPTCL(hw->port),
 916                            pf->stat_offsets_loaded,
 917                            &osd->eth.tx_unicast,
 918                            &nsd->eth.tx_unicast);
 919         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 920                            I40E_GLPRT_MPTCL(hw->port),
 921                            pf->stat_offsets_loaded,
 922                            &osd->eth.tx_multicast,
 923                            &nsd->eth.tx_multicast);
 924         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 925                            I40E_GLPRT_BPTCL(hw->port),
 926                            pf->stat_offsets_loaded,
 927                            &osd->eth.tx_broadcast,
 928                            &nsd->eth.tx_broadcast);
 929
 930         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 931                            pf->stat_offsets_loaded,
 932                            &osd->tx_dropped_link_down,
 933                            &nsd->tx_dropped_link_down);
 934
 935         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 936                            pf->stat_offsets_loaded,
 937                            &osd->crc_errors, &nsd->crc_errors);
 938
 939         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 940                            pf->stat_offsets_loaded,
 941                            &osd->illegal_bytes, &nsd->illegal_bytes);
 942
 943         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 944                            pf->stat_offsets_loaded,
 945                            &osd->mac_local_faults,
 946                            &nsd->mac_local_faults);
 947         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 948                            pf->stat_offsets_loaded,
 949                            &osd->mac_remote_faults,
 950                            &nsd->mac_remote_faults);
 951
 952         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 953                            pf->stat_offsets_loaded,
 954                            &osd->rx_length_errors,
 955                            &nsd->rx_length_errors);
 956
 957         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 958                            pf->stat_offsets_loaded,
 959                            &osd->link_xon_rx, &nsd->link_xon_rx);
 960         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 961                            pf->stat_offsets_loaded,
 962                            &osd->link_xon_tx, &nsd->link_xon_tx);
 963         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 964                            pf->stat_offsets_loaded,
 965                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 966         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 967                            pf->stat_offsets_loaded,
 968                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
 969
 970         for (i = 0; i < 8; i++) {
 971                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 972                                    pf->stat_offsets_loaded,
 973                                    &osd->priority_xoff_rx[i],
 974                                    &nsd->priority_xoff_rx[i]);
 975                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 976                                    pf->stat_offsets_loaded,
 977                                    &osd->priority_xon_rx[i],
 978                                    &nsd->priority_xon_rx[i]);
 979                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 980                                    pf->stat_offsets_loaded,
 981                                    &osd->priority_xon_tx[i],
 982                                    &nsd->priority_xon_tx[i]);
 983                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 984                                    pf->stat_offsets_loaded,
 985                                    &osd->priority_xoff_tx[i],
 986                                    &nsd->priority_xoff_tx[i]);
 987                 i40e_stat_update32(hw,
 988                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
 989                                    pf->stat_offsets_loaded,
 990                                    &osd->priority_xon_2_xoff[i],
 991                                    &nsd->priority_xon_2_xoff[i]);
 992         }
 993
 994         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
 995                            I40E_GLPRT_PRC64L(hw->port),
 996                            pf->stat_offsets_loaded,
 997                            &osd->rx_size_64, &nsd->rx_size_64);
 998         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
 999                            I40E_GLPRT_PRC127L(hw->port),
1000                            pf->stat_offsets_loaded,
1001                            &osd->rx_size_127, &nsd->rx_size_127);
1002         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1003                            I40E_GLPRT_PRC255L(hw->port),
1004                            pf->stat_offsets_loaded,
1005                            &osd->rx_size_255, &nsd->rx_size_255);
1006         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1007                            I40E_GLPRT_PRC511L(hw->port),
1008                            pf->stat_offsets_loaded,
1009                            &osd->rx_size_511, &nsd->rx_size_511);
1010         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1011                            I40E_GLPRT_PRC1023L(hw->port),
1012                            pf->stat_offsets_loaded,
1013                            &osd->rx_size_1023, &nsd->rx_size_1023);
1014         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1015                            I40E_GLPRT_PRC1522L(hw->port),
1016                            pf->stat_offsets_loaded,
1017                            &osd->rx_size_1522, &nsd->rx_size_1522);
1018         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1019                            I40E_GLPRT_PRC9522L(hw->port),
1020                            pf->stat_offsets_loaded,
1021                            &osd->rx_size_big, &nsd->rx_size_big);
1022
1023         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1024                            I40E_GLPRT_PTC64L(hw->port),
1025                            pf->stat_offsets_loaded,
1026                            &osd->tx_size_64, &nsd->tx_size_64);
1027         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1028                            I40E_GLPRT_PTC127L(hw->port),
1029                            pf->stat_offsets_loaded,
1030                            &osd->tx_size_127, &nsd->tx_size_127);
1031         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1032                            I40E_GLPRT_PTC255L(hw->port),
1033                            pf->stat_offsets_loaded,
1034                            &osd->tx_size_255, &nsd->tx_size_255);
1035         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1036                            I40E_GLPRT_PTC511L(hw->port),
1037                            pf->stat_offsets_loaded,
1038                            &osd->tx_size_511, &nsd->tx_size_511);
1039         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1040                            I40E_GLPRT_PTC1023L(hw->port),
1041                            pf->stat_offsets_loaded,
1042                            &osd->tx_size_1023, &nsd->tx_size_1023);
1043         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1044                            I40E_GLPRT_PTC1522L(hw->port),
1045                            pf->stat_offsets_loaded,
1046                            &osd->tx_size_1522, &nsd->tx_size_1522);
1047         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1048                            I40E_GLPRT_PTC9522L(hw->port),
1049                            pf->stat_offsets_loaded,
1050                            &osd->tx_size_big, &nsd->tx_size_big);
1051
1052         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1053                            pf->stat_offsets_loaded,
1054                            &osd->rx_undersize, &nsd->rx_undersize);
1055         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1056                            pf->stat_offsets_loaded,
1057                            &osd->rx_fragments, &nsd->rx_fragments);
1058         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1059                            pf->stat_offsets_loaded,
1060                            &osd->rx_oversize, &nsd->rx_oversize);
1061         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1062                            pf->stat_offsets_loaded,
1063                            &osd->rx_jabber, &nsd->rx_jabber);
1064
1065         /* FDIR stats */
1066         i40e_stat_update_and_clear32(hw,
1067                         I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1068                         &nsd->fd_atr_match);
1069         i40e_stat_update_and_clear32(hw,
1070                         I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1071                         &nsd->fd_sb_match);
1072         i40e_stat_update_and_clear32(hw,
1073                         I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1074                         &nsd->fd_atr_tunnel_match);
1075
1076         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1077         nsd->tx_lpi_status =
1078                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1079                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1080         nsd->rx_lpi_status =
1081                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1082                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1083         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1084                            pf->stat_offsets_loaded,
1085                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1086         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1087                            pf->stat_offsets_loaded,
1088                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1089
1090         if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1091             !(pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED))
1092                 nsd->fd_sb_status = true;
1093         else
1094                 nsd->fd_sb_status = false;
1095
1096         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1097             !(pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED))
1098                 nsd->fd_atr_status = true;
1099         else
1100                 nsd->fd_atr_status = false;
1101
1102         pf->stat_offsets_loaded = true;
1103 }
1104
1105 /**
1106  * i40e_update_stats - Update the various statistics counters.
1107  * @vsi: the VSI to be updated
1108  *
1109  * Update the various stats for this VSI and its related entities.
1110  **/
1111 void i40e_update_stats(struct i40e_vsi *vsi)
1112 {
1113         struct i40e_pf *pf = vsi->back;
1114
1115         if (vsi == pf->vsi[pf->lan_vsi])
1116                 i40e_update_pf_stats(pf);
1117
1118         i40e_update_vsi_stats(vsi);
1119 }
1120
1121 /**
1122  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1123  * @vsi: the VSI to be searched
1124  * @macaddr: the MAC address
1125  * @vlan: the vlan
1126  *
1127  * Returns ptr to the filter object or NULL
1128  **/
1129 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1130                                                 const u8 *macaddr, s16 vlan)
1131 {
1132         struct i40e_mac_filter *f;
1133         u64 key;
1134
1135         if (!vsi || !macaddr)
1136                 return NULL;
1137
1138         key = i40e_addr_to_hkey(macaddr);
1139         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1140                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1141                     (vlan == f->vlan))
1142                         return f;
1143         }
1144         return NULL;
1145 }
1146
1147 /**
1148  * i40e_find_mac - Find a mac addr in the macvlan filters list
1149  * @vsi: the VSI to be searched
1150  * @macaddr: the MAC address we are searching for
1151  *
1152  * Returns the first filter with the provided MAC address or NULL if
1153  * MAC address was not found
1154  **/
1155 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1156 {
1157         struct i40e_mac_filter *f;
1158         u64 key;
1159
1160         if (!vsi || !macaddr)
1161                 return NULL;
1162
1163         key = i40e_addr_to_hkey(macaddr);
1164         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1165                 if ((ether_addr_equal(macaddr, f->macaddr)))
1166                         return f;
1167         }
1168         return NULL;
1169 }
1170
1171 /**
1172  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1173  * @vsi: the VSI to be searched
1174  *
1175  * Returns true if VSI is in vlan mode or false otherwise
1176  **/
1177 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1178 {
1179         /* If we have a PVID, always operate in VLAN mode */
1180         if (vsi->info.pvid)
1181                 return true;
1182
1183         /* We need to operate in VLAN mode whenever we have any filters with
1184          * a VLAN other than I40E_VLAN_ALL. We could check the table each
1185          * time, incurring search cost repeatedly. However, we can notice two
1186          * things:
1187          *
1188          * 1) the only place where we can gain a VLAN filter is in
1189          *    i40e_add_filter.
1190          *
1191          * 2) the only place where filters are actually removed is in
1192          *    i40e_sync_filters_subtask.
1193          *
1194          * Thus, we can simply use a boolean value, has_vlan_filters which we
1195          * will set to true when we add a VLAN filter in i40e_add_filter. Then
1196          * we have to perform the full search after deleting filters in
1197          * i40e_sync_filters_subtask, but we already have to search
1198          * filters here and can perform the check at the same time. This
1199          * results in avoiding embedding a loop for VLAN mode inside another
1200          * loop over all the filters, and should maintain correctness as noted
1201          * above.
1202          */
1203         return vsi->has_vlan_filter;
1204 }
1205
1206 /**
1207  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1208  * @vsi: the VSI to configure
1209  * @tmp_add_list: list of filters ready to be added
1210  * @tmp_del_list: list of filters ready to be deleted
1211  * @vlan_filters: the number of active VLAN filters
1212  *
1213  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1214  * behave as expected. If we have any active VLAN filters remaining or about
1215  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1216  * so that they only match against untagged traffic. If we no longer have any
1217  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1218  * so that they match against both tagged and untagged traffic. In this way,
1219  * we ensure that we correctly receive the desired traffic. This ensures that
1220  * when we have an active VLAN we will receive only untagged traffic and
1221  * traffic matching active VLANs. If we have no active VLANs then we will
1222  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1223  *
1224  * Finally, in a similar fashion, this function also corrects filters when
1225  * there is an active PVID assigned to this VSI.
1226  *
1227  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1228  *
1229  * This function is only expected to be called from within
1230  * i40e_sync_vsi_filters.
1231  *
1232  * NOTE: This function expects to be called while under the
1233  * mac_filter_hash_lock
1234  */
1235 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1236                                          struct hlist_head *tmp_add_list,
1237                                          struct hlist_head *tmp_del_list,
1238                                          int vlan_filters)
1239 {
1240         s16 pvid = le16_to_cpu(vsi->info.pvid);
1241         struct i40e_mac_filter *f, *add_head;
1242         struct i40e_new_mac_filter *new;
1243         struct hlist_node *h;
1244         int bkt, new_vlan;
1245
1246         /* To determine if a particular filter needs to be replaced we
1247          * have the three following conditions:
1248          *
1249          * a) if we have a PVID assigned, then all filters which are
1250          *    not marked as VLAN=PVID must be replaced with filters that
1251          *    are.
1252          * b) otherwise, if we have any active VLANS, all filters
1253          *    which are marked as VLAN=-1 must be replaced with
1254          *    filters marked as VLAN=0
1255          * c) finally, if we do not have any active VLANS, all filters
1256          *    which are marked as VLAN=0 must be replaced with filters
1257          *    marked as VLAN=-1
1258          */
1259
1260         /* Update the filters about to be added in place */
1261         hlist_for_each_entry(new, tmp_add_list, hlist) {
1262                 if (pvid && new->f->vlan != pvid)
1263                         new->f->vlan = pvid;
1264                 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1265                         new->f->vlan = 0;
1266                 else if (!vlan_filters && new->f->vlan == 0)
1267                         new->f->vlan = I40E_VLAN_ANY;
1268         }
1269
1270         /* Update the remaining active filters */
1271         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1272                 /* Combine the checks for whether a filter needs to be changed
1273                  * and then determine the new VLAN inside the if block, in
1274                  * order to avoid duplicating code for adding the new filter
1275                  * then deleting the old filter.
1276                  */
1277                 if ((pvid && f->vlan != pvid) ||
1278                     (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1279                     (!vlan_filters && f->vlan == 0)) {
1280                         /* Determine the new vlan we will be adding */
1281                         if (pvid)
1282                                 new_vlan = pvid;
1283                         else if (vlan_filters)
1284                                 new_vlan = 0;
1285                         else
1286                                 new_vlan = I40E_VLAN_ANY;
1287
1288                         /* Create the new filter */
1289                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1290                         if (!add_head)
1291                                 return -ENOMEM;
1292
1293                         /* Create a temporary i40e_new_mac_filter */
1294                         new = kzalloc(sizeof(*new), GFP_ATOMIC);
1295                         if (!new)
1296                                 return -ENOMEM;
1297
1298                         new->f = add_head;
1299                         new->state = add_head->state;
1300
1301                         /* Add the new filter to the tmp list */
1302                         hlist_add_head(&new->hlist, tmp_add_list);
1303
1304                         /* Put the original filter into the delete list */
1305                         f->state = I40E_FILTER_REMOVE;
1306                         hash_del(&f->hlist);
1307                         hlist_add_head(&f->hlist, tmp_del_list);
1308                 }
1309         }
1310
1311         vsi->has_vlan_filter = !!vlan_filters;
1312
1313         return 0;
1314 }
1315
1316 /**
1317  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1318  * @vsi: the PF Main VSI - inappropriate for any other VSI
1319  * @macaddr: the MAC address
1320  *
1321  * Remove whatever filter the firmware set up so the driver can manage
1322  * its own filtering intelligently.
1323  **/
1324 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1325 {
1326         struct i40e_aqc_remove_macvlan_element_data element;
1327         struct i40e_pf *pf = vsi->back;
1328
1329         /* Only appropriate for the PF main VSI */
1330         if (vsi->type != I40E_VSI_MAIN)
1331                 return;
1332
1333         memset(&element, 0, sizeof(element));
1334         ether_addr_copy(element.mac_addr, macaddr);
1335         element.vlan_tag = 0;
1336         /* Ignore error returns, some firmware does it this way... */
1337         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1338         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1339
1340         memset(&element, 0, sizeof(element));
1341         ether_addr_copy(element.mac_addr, macaddr);
1342         element.vlan_tag = 0;
1343         /* ...and some firmware does it this way. */
1344         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1345                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1346         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1347 }
1348
1349 /**
1350  * i40e_add_filter - Add a mac/vlan filter to the VSI
1351  * @vsi: the VSI to be searched
1352  * @macaddr: the MAC address
1353  * @vlan: the vlan
1354  *
1355  * Returns ptr to the filter object or NULL when no memory available.
1356  *
1357  * NOTE: This function is expected to be called with mac_filter_hash_lock
1358  * being held.
1359  **/
1360 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1361                                         const u8 *macaddr, s16 vlan)
1362 {
1363         struct i40e_mac_filter *f;
1364         u64 key;
1365
1366         if (!vsi || !macaddr)
1367                 return NULL;
1368
1369         f = i40e_find_filter(vsi, macaddr, vlan);
1370         if (!f) {
1371                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1372                 if (!f)
1373                         return NULL;
1374
1375                 /* Update the boolean indicating if we need to function in
1376                  * VLAN mode.
1377                  */
1378                 if (vlan >= 0)
1379                         vsi->has_vlan_filter = true;
1380
1381                 ether_addr_copy(f->macaddr, macaddr);
1382                 f->vlan = vlan;
1383                 /* If we're in overflow promisc mode, set the state directly
1384                  * to failed, so we don't bother to try sending the filter
1385                  * to the hardware.
1386                  */
1387                 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state))
1388                         f->state = I40E_FILTER_FAILED;
1389                 else
1390                         f->state = I40E_FILTER_NEW;
1391                 INIT_HLIST_NODE(&f->hlist);
1392
1393                 key = i40e_addr_to_hkey(macaddr);
1394                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1395
1396                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1397                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1398         }
1399
1400         /* If we're asked to add a filter that has been marked for removal, it
1401          * is safe to simply restore it to active state. __i40e_del_filter
1402          * will have simply deleted any filters which were previously marked
1403          * NEW or FAILED, so if it is currently marked REMOVE it must have
1404          * previously been ACTIVE. Since we haven't yet run the sync filters
1405          * task, just restore this filter to the ACTIVE state so that the
1406          * sync task leaves it in place
1407          */
1408         if (f->state == I40E_FILTER_REMOVE)
1409                 f->state = I40E_FILTER_ACTIVE;
1410
1411         return f;
1412 }
1413
1414 /**
1415  * __i40e_del_filter - Remove a specific filter from the VSI
1416  * @vsi: VSI to remove from
1417  * @f: the filter to remove from the list
1418  *
1419  * This function should be called instead of i40e_del_filter only if you know
1420  * the exact filter you will remove already, such as via i40e_find_filter or
1421  * i40e_find_mac.
1422  *
1423  * NOTE: This function is expected to be called with mac_filter_hash_lock
1424  * being held.
1425  * ANOTHER NOTE: This function MUST be called from within the context of
1426  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1427  * instead of list_for_each_entry().
1428  **/
1429 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1430 {
1431         if (!f)
1432                 return;
1433
1434         /* If the filter was never added to firmware then we can just delete it
1435          * directly and we don't want to set the status to remove or else an
1436          * admin queue command will unnecessarily fire.
1437          */
1438         if ((f->state == I40E_FILTER_FAILED) ||
1439             (f->state == I40E_FILTER_NEW)) {
1440                 hash_del(&f->hlist);
1441                 kfree(f);
1442         } else {
1443                 f->state = I40E_FILTER_REMOVE;
1444         }
1445
1446         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1447         vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1448 }
1449
1450 /**
1451  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1452  * @vsi: the VSI to be searched
1453  * @macaddr: the MAC address
1454  * @vlan: the VLAN
1455  *
1456  * NOTE: This function is expected to be called with mac_filter_hash_lock
1457  * being held.
1458  * ANOTHER NOTE: This function MUST be called from within the context of
1459  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1460  * instead of list_for_each_entry().
1461  **/
1462 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1463 {
1464         struct i40e_mac_filter *f;
1465
1466         if (!vsi || !macaddr)
1467                 return;
1468
1469         f = i40e_find_filter(vsi, macaddr, vlan);
1470         __i40e_del_filter(vsi, f);
1471 }
1472
1473 /**
1474  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1475  * @vsi: the VSI to be searched
1476  * @macaddr: the mac address to be filtered
1477  *
1478  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1479  * go through all the macvlan filters and add a macvlan filter for each
1480  * unique vlan that already exists. If a PVID has been assigned, instead only
1481  * add the macaddr to that VLAN.
1482  *
1483  * Returns last filter added on success, else NULL
1484  **/
1485 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1486                                             const u8 *macaddr)
1487 {
1488         struct i40e_mac_filter *f, *add = NULL;
1489         struct hlist_node *h;
1490         int bkt;
1491
1492         if (vsi->info.pvid)
1493                 return i40e_add_filter(vsi, macaddr,
1494                                        le16_to_cpu(vsi->info.pvid));
1495
1496         if (!i40e_is_vsi_in_vlan(vsi))
1497                 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1498
1499         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1500                 if (f->state == I40E_FILTER_REMOVE)
1501                         continue;
1502                 add = i40e_add_filter(vsi, macaddr, f->vlan);
1503                 if (!add)
1504                         return NULL;
1505         }
1506
1507         return add;
1508 }
1509
1510 /**
1511  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1512  * @vsi: the VSI to be searched
1513  * @macaddr: the mac address to be removed
1514  *
1515  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1516  * associated with.
1517  *
1518  * Returns 0 for success, or error
1519  **/
1520 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1521 {
1522         struct i40e_mac_filter *f;
1523         struct hlist_node *h;
1524         bool found = false;
1525         int bkt;
1526
1527         WARN(!spin_is_locked(&vsi->mac_filter_hash_lock),
1528              "Missing mac_filter_hash_lock\n");
1529         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1530                 if (ether_addr_equal(macaddr, f->macaddr)) {
1531                         __i40e_del_filter(vsi, f);
1532                         found = true;
1533                 }
1534         }
1535
1536         if (found)
1537                 return 0;
1538         else
1539                 return -ENOENT;
1540 }
1541
1542 /**
1543  * i40e_set_mac - NDO callback to set mac address
1544  * @netdev: network interface device structure
1545  * @p: pointer to an address structure
1546  *
1547  * Returns 0 on success, negative on failure
1548  **/
1549 static int i40e_set_mac(struct net_device *netdev, void *p)
1550 {
1551         struct i40e_netdev_priv *np = netdev_priv(netdev);
1552         struct i40e_vsi *vsi = np->vsi;
1553         struct i40e_pf *pf = vsi->back;
1554         struct i40e_hw *hw = &pf->hw;
1555         struct sockaddr *addr = p;
1556
1557         if (!is_valid_ether_addr(addr->sa_data))
1558                 return -EADDRNOTAVAIL;
1559
1560         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1561                 netdev_info(netdev, "already using mac address %pM\n",
1562                             addr->sa_data);
1563                 return 0;
1564         }
1565
1566         if (test_bit(__I40E_VSI_DOWN, vsi->back->state) ||
1567             test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state))
1568                 return -EADDRNOTAVAIL;
1569
1570         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1571                 netdev_info(netdev, "returning to hw mac address %pM\n",
1572                             hw->mac.addr);
1573         else
1574                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1575
1576         /* Copy the address first, so that we avoid a possible race with
1577          * .set_rx_mode(). If we copy after changing the address in the filter
1578          * list, we might open ourselves to a narrow race window where
1579          * .set_rx_mode could delete our dev_addr filter and prevent traffic
1580          * from passing.
1581          */
1582         ether_addr_copy(netdev->dev_addr, addr->sa_data);
1583
1584         spin_lock_bh(&vsi->mac_filter_hash_lock);
1585         i40e_del_mac_filter(vsi, netdev->dev_addr);
1586         i40e_add_mac_filter(vsi, addr->sa_data);
1587         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1588         if (vsi->type == I40E_VSI_MAIN) {
1589                 i40e_status ret;
1590
1591                 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1592                                                 I40E_AQC_WRITE_TYPE_LAA_WOL,
1593                                                 addr->sa_data, NULL);
1594                 if (ret)
1595                         netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1596                                     i40e_stat_str(hw, ret),
1597                                     i40e_aq_str(hw, hw->aq.asq_last_status));
1598         }
1599
1600         /* schedule our worker thread which will take care of
1601          * applying the new filter changes
1602          */
1603         i40e_service_event_schedule(vsi->back);
1604         return 0;
1605 }
1606
1607 /**
1608  * i40e_config_rss_aq - Prepare for RSS using AQ commands
1609  * @vsi: vsi structure
1610  * @seed: RSS hash seed
1611  **/
1612 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1613                               u8 *lut, u16 lut_size)
1614 {
1615         struct i40e_pf *pf = vsi->back;
1616         struct i40e_hw *hw = &pf->hw;
1617         int ret = 0;
1618
1619         if (seed) {
1620                 struct i40e_aqc_get_set_rss_key_data *seed_dw =
1621                         (struct i40e_aqc_get_set_rss_key_data *)seed;
1622                 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1623                 if (ret) {
1624                         dev_info(&pf->pdev->dev,
1625                                  "Cannot set RSS key, err %s aq_err %s\n",
1626                                  i40e_stat_str(hw, ret),
1627                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1628                         return ret;
1629                 }
1630         }
1631         if (lut) {
1632                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
1633
1634                 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1635                 if (ret) {
1636                         dev_info(&pf->pdev->dev,
1637                                  "Cannot set RSS lut, err %s aq_err %s\n",
1638                                  i40e_stat_str(hw, ret),
1639                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1640                         return ret;
1641                 }
1642         }
1643         return ret;
1644 }
1645
1646 /**
1647  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1648  * @vsi: VSI structure
1649  **/
1650 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1651 {
1652         struct i40e_pf *pf = vsi->back;
1653         u8 seed[I40E_HKEY_ARRAY_SIZE];
1654         u8 *lut;
1655         int ret;
1656
1657         if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
1658                 return 0;
1659         if (!vsi->rss_size)
1660                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
1661                                       vsi->num_queue_pairs);
1662         if (!vsi->rss_size)
1663                 return -EINVAL;
1664         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1665         if (!lut)
1666                 return -ENOMEM;
1667
1668         /* Use the user configured hash keys and lookup table if there is one,
1669          * otherwise use default
1670          */
1671         if (vsi->rss_lut_user)
1672                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1673         else
1674                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1675         if (vsi->rss_hkey_user)
1676                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1677         else
1678                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1679         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1680         kfree(lut);
1681         return ret;
1682 }
1683
1684 /**
1685  * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1686  * @vsi: the VSI being configured,
1687  * @ctxt: VSI context structure
1688  * @enabled_tc: number of traffic classes to enable
1689  *
1690  * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1691  **/
1692 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1693                                            struct i40e_vsi_context *ctxt,
1694                                            u8 enabled_tc)
1695 {
1696         u16 qcount = 0, max_qcount, qmap, sections = 0;
1697         int i, override_q, pow, num_qps, ret;
1698         u8 netdev_tc = 0, offset = 0;
1699
1700         if (vsi->type != I40E_VSI_MAIN)
1701                 return -EINVAL;
1702         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1703         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1704         vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1705         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1706         num_qps = vsi->mqprio_qopt.qopt.count[0];
1707
1708         /* find the next higher power-of-2 of num queue pairs */
1709         pow = ilog2(num_qps);
1710         if (!is_power_of_2(num_qps))
1711                 pow++;
1712         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1713                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1714
1715         /* Setup queue offset/count for all TCs for given VSI */
1716         max_qcount = vsi->mqprio_qopt.qopt.count[0];
1717         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1718                 /* See if the given TC is enabled for the given VSI */
1719                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1720                         offset = vsi->mqprio_qopt.qopt.offset[i];
1721                         qcount = vsi->mqprio_qopt.qopt.count[i];
1722                         if (qcount > max_qcount)
1723                                 max_qcount = qcount;
1724                         vsi->tc_config.tc_info[i].qoffset = offset;
1725                         vsi->tc_config.tc_info[i].qcount = qcount;
1726                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1727                 } else {
1728                         /* TC is not enabled so set the offset to
1729                          * default queue and allocate one queue
1730                          * for the given TC.
1731                          */
1732                         vsi->tc_config.tc_info[i].qoffset = 0;
1733                         vsi->tc_config.tc_info[i].qcount = 1;
1734                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1735                 }
1736         }
1737
1738         /* Set actual Tx/Rx queue pairs */
1739         vsi->num_queue_pairs = offset + qcount;
1740
1741         /* Setup queue TC[0].qmap for given VSI context */
1742         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1743         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1744         ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1745         ctxt->info.valid_sections |= cpu_to_le16(sections);
1746
1747         /* Reconfigure RSS for main VSI with max queue count */
1748         vsi->rss_size = max_qcount;
1749         ret = i40e_vsi_config_rss(vsi);
1750         if (ret) {
1751                 dev_info(&vsi->back->pdev->dev,
1752                          "Failed to reconfig rss for num_queues (%u)\n",
1753                          max_qcount);
1754                 return ret;
1755         }
1756         vsi->reconfig_rss = true;
1757         dev_dbg(&vsi->back->pdev->dev,
1758                 "Reconfigured rss with num_queues (%u)\n", max_qcount);
1759
1760         /* Find queue count available for channel VSIs and starting offset
1761          * for channel VSIs
1762          */
1763         override_q = vsi->mqprio_qopt.qopt.count[0];
1764         if (override_q && override_q < vsi->num_queue_pairs) {
1765                 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
1766                 vsi->next_base_queue = override_q;
1767         }
1768         return 0;
1769 }
1770
1771 /**
1772  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1773  * @vsi: the VSI being setup
1774  * @ctxt: VSI context structure
1775  * @enabled_tc: Enabled TCs bitmap
1776  * @is_add: True if called before Add VSI
1777  *
1778  * Setup VSI queue mapping for enabled traffic classes.
1779  **/
1780 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1781                                      struct i40e_vsi_context *ctxt,
1782                                      u8 enabled_tc,
1783                                      bool is_add)
1784 {
1785         struct i40e_pf *pf = vsi->back;
1786         u16 sections = 0;
1787         u8 netdev_tc = 0;
1788         u16 numtc = 0;
1789         u16 qcount;
1790         u8 offset;
1791         u16 qmap;
1792         int i;
1793         u16 num_tc_qps = 0;
1794
1795         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1796         offset = 0;
1797
1798         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1799                 /* Find numtc from enabled TC bitmap */
1800                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1801                         if (enabled_tc & BIT(i)) /* TC is enabled */
1802                                 numtc++;
1803                 }
1804                 if (!numtc) {
1805                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1806                         numtc = 1;
1807                 }
1808         } else {
1809                 /* At least TC0 is enabled in non-DCB, non-MQPRIO case */
1810                 numtc = 1;
1811         }
1812
1813         vsi->tc_config.numtc = numtc;
1814         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1815         /* Number of queues per enabled TC */
1816         qcount = vsi->alloc_queue_pairs;
1817
1818         num_tc_qps = qcount / numtc;
1819         num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf));
1820
1821         /* Setup queue offset/count for all TCs for given VSI */
1822         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1823                 /* See if the given TC is enabled for the given VSI */
1824                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1825                         /* TC is enabled */
1826                         int pow, num_qps;
1827
1828                         switch (vsi->type) {
1829                         case I40E_VSI_MAIN:
1830                                 qcount = min_t(int, pf->alloc_rss_size,
1831                                                num_tc_qps);
1832                                 break;
1833                         case I40E_VSI_FDIR:
1834                         case I40E_VSI_SRIOV:
1835                         case I40E_VSI_VMDQ2:
1836                         default:
1837                                 qcount = num_tc_qps;
1838                                 WARN_ON(i != 0);
1839                                 break;
1840                         }
1841                         vsi->tc_config.tc_info[i].qoffset = offset;
1842                         vsi->tc_config.tc_info[i].qcount = qcount;
1843
1844                         /* find the next higher power-of-2 of num queue pairs */
1845                         num_qps = qcount;
1846                         pow = 0;
1847                         while (num_qps && (BIT_ULL(pow) < qcount)) {
1848                                 pow++;
1849                                 num_qps >>= 1;
1850                         }
1851
1852                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1853                         qmap =
1854                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1855                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1856
1857                         offset += qcount;
1858                 } else {
1859                         /* TC is not enabled so set the offset to
1860                          * default queue and allocate one queue
1861                          * for the given TC.
1862                          */
1863                         vsi->tc_config.tc_info[i].qoffset = 0;
1864                         vsi->tc_config.tc_info[i].qcount = 1;
1865                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1866
1867                         qmap = 0;
1868                 }
1869                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1870         }
1871
1872         /* Set actual Tx/Rx queue pairs */
1873         vsi->num_queue_pairs = offset;
1874         if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) {
1875                 if (vsi->req_queue_pairs > 0)
1876                         vsi->num_queue_pairs = vsi->req_queue_pairs;
1877                 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1878                         vsi->num_queue_pairs = pf->num_lan_msix;
1879         }
1880
1881         /* Scheduler section valid can only be set for ADD VSI */
1882         if (is_add) {
1883                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1884
1885                 ctxt->info.up_enable_bits = enabled_tc;
1886         }
1887         if (vsi->type == I40E_VSI_SRIOV) {
1888                 ctxt->info.mapping_flags |=
1889                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1890                 for (i = 0; i < vsi->num_queue_pairs; i++)
1891                         ctxt->info.queue_mapping[i] =
1892                                                cpu_to_le16(vsi->base_queue + i);
1893         } else {
1894                 ctxt->info.mapping_flags |=
1895                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1896                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1897         }
1898         ctxt->info.valid_sections |= cpu_to_le16(sections);
1899 }
1900
1901 /**
1902  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
1903  * @netdev: the netdevice
1904  * @addr: address to add
1905  *
1906  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1907  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1908  */
1909 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
1910 {
1911         struct i40e_netdev_priv *np = netdev_priv(netdev);
1912         struct i40e_vsi *vsi = np->vsi;
1913
1914         if (i40e_add_mac_filter(vsi, addr))
1915                 return 0;
1916         else
1917                 return -ENOMEM;
1918 }
1919
1920 /**
1921  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1922  * @netdev: the netdevice
1923  * @addr: address to add
1924  *
1925  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1926  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1927  */
1928 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
1929 {
1930         struct i40e_netdev_priv *np = netdev_priv(netdev);
1931         struct i40e_vsi *vsi = np->vsi;
1932
1933         /* Under some circumstances, we might receive a request to delete
1934          * our own device address from our uc list. Because we store the
1935          * device address in the VSI's MAC/VLAN filter list, we need to ignore
1936          * such requests and not delete our device address from this list.
1937          */
1938         if (ether_addr_equal(addr, netdev->dev_addr))
1939                 return 0;
1940
1941         i40e_del_mac_filter(vsi, addr);
1942
1943         return 0;
1944 }
1945
1946 /**
1947  * i40e_set_rx_mode - NDO callback to set the netdev filters
1948  * @netdev: network interface device structure
1949  **/
1950 static void i40e_set_rx_mode(struct net_device *netdev)
1951 {
1952         struct i40e_netdev_priv *np = netdev_priv(netdev);
1953         struct i40e_vsi *vsi = np->vsi;
1954
1955         spin_lock_bh(&vsi->mac_filter_hash_lock);
1956
1957         __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1958         __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1959
1960         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1961
1962         /* check for other flag changes */
1963         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1964                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1965                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1966         }
1967 }
1968
1969 /**
1970  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
1971  * @vsi: Pointer to VSI struct
1972  * @from: Pointer to list which contains MAC filter entries - changes to
1973  *        those entries needs to be undone.
1974  *
1975  * MAC filter entries from this list were slated for deletion.
1976  **/
1977 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
1978                                          struct hlist_head *from)
1979 {
1980         struct i40e_mac_filter *f;
1981         struct hlist_node *h;
1982
1983         hlist_for_each_entry_safe(f, h, from, hlist) {
1984                 u64 key = i40e_addr_to_hkey(f->macaddr);
1985
1986                 /* Move the element back into MAC filter list*/
1987                 hlist_del(&f->hlist);
1988                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1989         }
1990 }
1991
1992 /**
1993  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
1994  * @vsi: Pointer to vsi struct
1995  * @from: Pointer to list which contains MAC filter entries - changes to
1996  *        those entries needs to be undone.
1997  *
1998  * MAC filter entries from this list were slated for addition.
1999  **/
2000 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2001                                          struct hlist_head *from)
2002 {
2003         struct i40e_new_mac_filter *new;
2004         struct hlist_node *h;
2005
2006         hlist_for_each_entry_safe(new, h, from, hlist) {
2007                 /* We can simply free the wrapper structure */
2008                 hlist_del(&new->hlist);
2009                 kfree(new);
2010         }
2011 }
2012
2013 /**
2014  * i40e_next_entry - Get the next non-broadcast filter from a list
2015  * @next: pointer to filter in list
2016  *
2017  * Returns the next non-broadcast filter in the list. Required so that we
2018  * ignore broadcast filters within the list, since these are not handled via
2019  * the normal firmware update path.
2020  */
2021 static
2022 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2023 {
2024         hlist_for_each_entry_continue(next, hlist) {
2025                 if (!is_broadcast_ether_addr(next->f->macaddr))
2026                         return next;
2027         }
2028
2029         return NULL;
2030 }
2031
2032 /**
2033  * i40e_update_filter_state - Update filter state based on return data
2034  * from firmware
2035  * @count: Number of filters added
2036  * @add_list: return data from fw
2037  * @head: pointer to first filter in current batch
2038  *
2039  * MAC filter entries from list were slated to be added to device. Returns
2040  * number of successful filters. Note that 0 does NOT mean success!
2041  **/
2042 static int
2043 i40e_update_filter_state(int count,
2044                          struct i40e_aqc_add_macvlan_element_data *add_list,
2045                          struct i40e_new_mac_filter *add_head)
2046 {
2047         int retval = 0;
2048         int i;
2049
2050         for (i = 0; i < count; i++) {
2051                 /* Always check status of each filter. We don't need to check
2052                  * the firmware return status because we pre-set the filter
2053                  * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2054                  * request to the adminq. Thus, if it no longer matches then
2055                  * we know the filter is active.
2056                  */
2057                 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2058                         add_head->state = I40E_FILTER_FAILED;
2059                 } else {
2060                         add_head->state = I40E_FILTER_ACTIVE;
2061                         retval++;
2062                 }
2063
2064                 add_head = i40e_next_filter(add_head);
2065                 if (!add_head)
2066                         break;
2067         }
2068
2069         return retval;
2070 }
2071
2072 /**
2073  * i40e_aqc_del_filters - Request firmware to delete a set of filters
2074  * @vsi: ptr to the VSI
2075  * @vsi_name: name to display in messages
2076  * @list: the list of filters to send to firmware
2077  * @num_del: the number of filters to delete
2078  * @retval: Set to -EIO on failure to delete
2079  *
2080  * Send a request to firmware via AdminQ to delete a set of filters. Uses
2081  * *retval instead of a return value so that success does not force ret_val to
2082  * be set to 0. This ensures that a sequence of calls to this function
2083  * preserve the previous value of *retval on successful delete.
2084  */
2085 static
2086 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2087                           struct i40e_aqc_remove_macvlan_element_data *list,
2088                           int num_del, int *retval)
2089 {
2090         struct i40e_hw *hw = &vsi->back->hw;
2091         i40e_status aq_ret;
2092         int aq_err;
2093
2094         aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
2095         aq_err = hw->aq.asq_last_status;
2096
2097         /* Explicitly ignore and do not report when firmware returns ENOENT */
2098         if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
2099                 *retval = -EIO;
2100                 dev_info(&vsi->back->pdev->dev,
2101                          "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
2102                          vsi_name, i40e_stat_str(hw, aq_ret),
2103                          i40e_aq_str(hw, aq_err));
2104         }
2105 }
2106
2107 /**
2108  * i40e_aqc_add_filters - Request firmware to add a set of filters
2109  * @vsi: ptr to the VSI
2110  * @vsi_name: name to display in messages
2111  * @list: the list of filters to send to firmware
2112  * @add_head: Position in the add hlist
2113  * @num_add: the number of filters to add
2114  * @promisc_change: set to true on exit if promiscuous mode was forced on
2115  *
2116  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
2117  * promisc_changed to true if the firmware has run out of space for more
2118  * filters.
2119  */
2120 static
2121 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2122                           struct i40e_aqc_add_macvlan_element_data *list,
2123                           struct i40e_new_mac_filter *add_head,
2124                           int num_add, bool *promisc_changed)
2125 {
2126         struct i40e_hw *hw = &vsi->back->hw;
2127         int aq_err, fcnt;
2128
2129         i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
2130         aq_err = hw->aq.asq_last_status;
2131         fcnt = i40e_update_filter_state(num_add, list, add_head);
2132
2133         if (fcnt != num_add) {
2134                 *promisc_changed = true;
2135                 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2136                 dev_warn(&vsi->back->pdev->dev,
2137                          "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2138                          i40e_aq_str(hw, aq_err),
2139                          vsi_name);
2140         }
2141 }
2142
2143 /**
2144  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2145  * @vsi: pointer to the VSI
2146  * @f: filter data
2147  *
2148  * This function sets or clears the promiscuous broadcast flags for VLAN
2149  * filters in order to properly receive broadcast frames. Assumes that only
2150  * broadcast filters are passed.
2151  *
2152  * Returns status indicating success or failure;
2153  **/
2154 static i40e_status
2155 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2156                           struct i40e_mac_filter *f)
2157 {
2158         bool enable = f->state == I40E_FILTER_NEW;
2159         struct i40e_hw *hw = &vsi->back->hw;
2160         i40e_status aq_ret;
2161
2162         if (f->vlan == I40E_VLAN_ANY) {
2163                 aq_ret = i40e_aq_set_vsi_broadcast(hw,
2164                                                    vsi->seid,
2165                                                    enable,
2166                                                    NULL);
2167         } else {
2168                 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2169                                                             vsi->seid,
2170                                                             enable,
2171                                                             f->vlan,
2172                                                             NULL);
2173         }
2174
2175         if (aq_ret)
2176                 dev_warn(&vsi->back->pdev->dev,
2177                          "Error %s setting broadcast promiscuous mode on %s\n",
2178                          i40e_aq_str(hw, hw->aq.asq_last_status),
2179                          vsi_name);
2180
2181         return aq_ret;
2182 }
2183
2184 /**
2185  * i40e_set_promiscuous - set promiscuous mode
2186  * @pf: board private structure
2187  * @promisc: promisc on or off
2188  *
2189  * There are different ways of setting promiscuous mode on a PF depending on
2190  * what state/environment we're in.  This identifies and sets it appropriately.
2191  * Returns 0 on success.
2192  **/
2193 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2194 {
2195         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
2196         struct i40e_hw *hw = &pf->hw;
2197         i40e_status aq_ret;
2198
2199         if (vsi->type == I40E_VSI_MAIN &&
2200             pf->lan_veb != I40E_NO_VEB &&
2201             !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2202                 /* set defport ON for Main VSI instead of true promisc
2203                  * this way we will get all unicast/multicast and VLAN
2204                  * promisc behavior but will not get VF or VMDq traffic
2205                  * replicated on the Main VSI.
2206                  */
2207                 if (promisc)
2208                         aq_ret = i40e_aq_set_default_vsi(hw,
2209                                                          vsi->seid,
2210                                                          NULL);
2211                 else
2212                         aq_ret = i40e_aq_clear_default_vsi(hw,
2213                                                            vsi->seid,
2214                                                            NULL);
2215                 if (aq_ret) {
2216                         dev_info(&pf->pdev->dev,
2217                                  "Set default VSI failed, err %s, aq_err %s\n",
2218                                  i40e_stat_str(hw, aq_ret),
2219                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2220                 }
2221         } else {
2222                 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2223                                                   hw,
2224                                                   vsi->seid,
2225                                                   promisc, NULL,
2226                                                   true);
2227                 if (aq_ret) {
2228                         dev_info(&pf->pdev->dev,
2229                                  "set unicast promisc failed, err %s, aq_err %s\n",
2230                                  i40e_stat_str(hw, aq_ret),
2231                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2232                 }
2233                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2234                                                   hw,
2235                                                   vsi->seid,
2236                                                   promisc, NULL);
2237                 if (aq_ret) {
2238                         dev_info(&pf->pdev->dev,
2239                                  "set multicast promisc failed, err %s, aq_err %s\n",
2240                                  i40e_stat_str(hw, aq_ret),
2241                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2242                 }
2243         }
2244
2245         if (!aq_ret)
2246                 pf->cur_promisc = promisc;
2247
2248         return aq_ret;
2249 }
2250
2251 /**
2252  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2253  * @vsi: ptr to the VSI
2254  *
2255  * Push any outstanding VSI filter changes through the AdminQ.
2256  *
2257  * Returns 0 or error value
2258  **/
2259 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2260 {
2261         struct hlist_head tmp_add_list, tmp_del_list;
2262         struct i40e_mac_filter *f;
2263         struct i40e_new_mac_filter *new, *add_head = NULL;
2264         struct i40e_hw *hw = &vsi->back->hw;
2265         unsigned int failed_filters = 0;
2266         unsigned int vlan_filters = 0;
2267         bool promisc_changed = false;
2268         char vsi_name[16] = "PF";
2269         int filter_list_len = 0;
2270         i40e_status aq_ret = 0;
2271         u32 changed_flags = 0;
2272         struct hlist_node *h;
2273         struct i40e_pf *pf;
2274         int num_add = 0;
2275         int num_del = 0;
2276         int retval = 0;
2277         u16 cmd_flags;
2278         int list_size;
2279         int bkt;
2280
2281         /* empty array typed pointers, kcalloc later */
2282         struct i40e_aqc_add_macvlan_element_data *add_list;
2283         struct i40e_aqc_remove_macvlan_element_data *del_list;
2284
2285         while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2286                 usleep_range(1000, 2000);
2287         pf = vsi->back;
2288
2289         if (vsi->netdev) {
2290                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2291                 vsi->current_netdev_flags = vsi->netdev->flags;
2292         }
2293
2294         INIT_HLIST_HEAD(&tmp_add_list);
2295         INIT_HLIST_HEAD(&tmp_del_list);
2296
2297         if (vsi->type == I40E_VSI_SRIOV)
2298                 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2299         else if (vsi->type != I40E_VSI_MAIN)
2300                 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2301
2302         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2303                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2304
2305                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2306                 /* Create a list of filters to delete. */
2307                 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2308                         if (f->state == I40E_FILTER_REMOVE) {
2309                                 /* Move the element into temporary del_list */
2310                                 hash_del(&f->hlist);
2311                                 hlist_add_head(&f->hlist, &tmp_del_list);
2312
2313                                 /* Avoid counting removed filters */
2314                                 continue;
2315                         }
2316                         if (f->state == I40E_FILTER_NEW) {
2317                                 /* Create a temporary i40e_new_mac_filter */
2318                                 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2319                                 if (!new)
2320                                         goto err_no_memory_locked;
2321
2322                                 /* Store pointer to the real filter */
2323                                 new->f = f;
2324                                 new->state = f->state;
2325
2326                                 /* Add it to the hash list */
2327                                 hlist_add_head(&new->hlist, &tmp_add_list);
2328                         }
2329
2330                         /* Count the number of active (current and new) VLAN
2331                          * filters we have now. Does not count filters which
2332                          * are marked for deletion.
2333                          */
2334                         if (f->vlan > 0)
2335                                 vlan_filters++;
2336                 }
2337
2338                 retval = i40e_correct_mac_vlan_filters(vsi,
2339                                                        &tmp_add_list,
2340                                                        &tmp_del_list,
2341                                                        vlan_filters);
2342                 if (retval)
2343                         goto err_no_memory_locked;
2344
2345                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2346         }
2347
2348         /* Now process 'del_list' outside the lock */
2349         if (!hlist_empty(&tmp_del_list)) {
2350                 filter_list_len = hw->aq.asq_buf_size /
2351                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2352                 list_size = filter_list_len *
2353                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2354                 del_list = kzalloc(list_size, GFP_ATOMIC);
2355                 if (!del_list)
2356                         goto err_no_memory;
2357
2358                 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2359                         cmd_flags = 0;
2360
2361                         /* handle broadcast filters by updating the broadcast
2362                          * promiscuous flag and release filter list.
2363                          */
2364                         if (is_broadcast_ether_addr(f->macaddr)) {
2365                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2366
2367                                 hlist_del(&f->hlist);
2368                                 kfree(f);
2369                                 continue;
2370                         }
2371
2372                         /* add to delete list */
2373                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2374                         if (f->vlan == I40E_VLAN_ANY) {
2375                                 del_list[num_del].vlan_tag = 0;
2376                                 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2377                         } else {
2378                                 del_list[num_del].vlan_tag =
2379                                         cpu_to_le16((u16)(f->vlan));
2380                         }
2381
2382                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2383                         del_list[num_del].flags = cmd_flags;
2384                         num_del++;
2385
2386                         /* flush a full buffer */
2387                         if (num_del == filter_list_len) {
2388                                 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2389                                                      num_del, &retval);
2390                                 memset(del_list, 0, list_size);
2391                                 num_del = 0;
2392                         }
2393                         /* Release memory for MAC filter entries which were
2394                          * synced up with HW.
2395                          */
2396                         hlist_del(&f->hlist);
2397                         kfree(f);
2398                 }
2399
2400                 if (num_del) {
2401                         i40e_aqc_del_filters(vsi, vsi_name, del_list,
2402                                              num_del, &retval);
2403                 }
2404
2405                 kfree(del_list);
2406                 del_list = NULL;
2407         }
2408
2409         if (!hlist_empty(&tmp_add_list)) {
2410                 /* Do all the adds now. */
2411                 filter_list_len = hw->aq.asq_buf_size /
2412                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2413                 list_size = filter_list_len *
2414                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2415                 add_list = kzalloc(list_size, GFP_ATOMIC);
2416                 if (!add_list)
2417                         goto err_no_memory;
2418
2419                 num_add = 0;
2420                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2421                         if (test_bit(__I40E_VSI_OVERFLOW_PROMISC,
2422                                      vsi->state)) {
2423                                 new->state = I40E_FILTER_FAILED;
2424                                 continue;
2425                         }
2426
2427                         /* handle broadcast filters by updating the broadcast
2428                          * promiscuous flag instead of adding a MAC filter.
2429                          */
2430                         if (is_broadcast_ether_addr(new->f->macaddr)) {
2431                                 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2432                                                               new->f))
2433                                         new->state = I40E_FILTER_FAILED;
2434                                 else
2435                                         new->state = I40E_FILTER_ACTIVE;
2436                                 continue;
2437                         }
2438
2439                         /* add to add array */
2440                         if (num_add == 0)
2441                                 add_head = new;
2442                         cmd_flags = 0;
2443                         ether_addr_copy(add_list[num_add].mac_addr,
2444                                         new->f->macaddr);
2445                         if (new->f->vlan == I40E_VLAN_ANY) {
2446                                 add_list[num_add].vlan_tag = 0;
2447                                 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2448                         } else {
2449                                 add_list[num_add].vlan_tag =
2450                                         cpu_to_le16((u16)(new->f->vlan));
2451                         }
2452                         add_list[num_add].queue_number = 0;
2453                         /* set invalid match method for later detection */
2454                         add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2455                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2456                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2457                         num_add++;
2458
2459                         /* flush a full buffer */
2460                         if (num_add == filter_list_len) {
2461                                 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2462                                                      add_head, num_add,
2463                                                      &promisc_changed);
2464                                 memset(add_list, 0, list_size);
2465                                 num_add = 0;
2466                         }
2467                 }
2468                 if (num_add) {
2469                         i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2470                                              num_add, &promisc_changed);
2471                 }
2472                 /* Now move all of the filters from the temp add list back to
2473                  * the VSI's list.
2474                  */
2475                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2476                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2477                         /* Only update the state if we're still NEW */
2478                         if (new->f->state == I40E_FILTER_NEW)
2479                                 new->f->state = new->state;
2480                         hlist_del(&new->hlist);
2481                         kfree(new);
2482                 }
2483                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2484                 kfree(add_list);
2485                 add_list = NULL;
2486         }
2487
2488         /* Determine the number of active and failed filters. */
2489         spin_lock_bh(&vsi->mac_filter_hash_lock);
2490         vsi->active_filters = 0;
2491         hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2492                 if (f->state == I40E_FILTER_ACTIVE)
2493                         vsi->active_filters++;
2494                 else if (f->state == I40E_FILTER_FAILED)
2495                         failed_filters++;
2496         }
2497         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2498
2499         /* If promiscuous mode has changed, we need to calculate a new
2500          * threshold for when we are safe to exit
2501          */
2502         if (promisc_changed)
2503                 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2504
2505         /* Check if we are able to exit overflow promiscuous mode. We can
2506          * safely exit if we didn't just enter, we no longer have any failed
2507          * filters, and we have reduced filters below the threshold value.
2508          */
2509         if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state) &&
2510             !promisc_changed && !failed_filters &&
2511             (vsi->active_filters < vsi->promisc_threshold)) {
2512                 dev_info(&pf->pdev->dev,
2513                          "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2514                          vsi_name);
2515                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2516                 promisc_changed = true;
2517                 vsi->promisc_threshold = 0;
2518         }
2519
2520         /* if the VF is not trusted do not do promisc */
2521         if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2522                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2523                 goto out;
2524         }
2525
2526         /* check for changes in promiscuous modes */
2527         if (changed_flags & IFF_ALLMULTI) {
2528                 bool cur_multipromisc;
2529
2530                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2531                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2532                                                                vsi->seid,
2533                                                                cur_multipromisc,
2534                                                                NULL);
2535                 if (aq_ret) {
2536                         retval = i40e_aq_rc_to_posix(aq_ret,
2537                                                      hw->aq.asq_last_status);
2538                         dev_info(&pf->pdev->dev,
2539                                  "set multi promisc failed on %s, err %s aq_err %s\n",
2540                                  vsi_name,
2541                                  i40e_stat_str(hw, aq_ret),
2542                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2543                 }
2544         }
2545
2546         if ((changed_flags & IFF_PROMISC) || promisc_changed) {
2547                 bool cur_promisc;
2548
2549                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2550                                test_bit(__I40E_VSI_OVERFLOW_PROMISC,
2551                                         vsi->state));
2552                 aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2553                 if (aq_ret) {
2554                         retval = i40e_aq_rc_to_posix(aq_ret,
2555                                                      hw->aq.asq_last_status);
2556                         dev_info(&pf->pdev->dev,
2557                                  "Setting promiscuous %s failed on %s, err %s aq_err %s\n",
2558                                  cur_promisc ? "on" : "off",
2559                                  vsi_name,
2560                                  i40e_stat_str(hw, aq_ret),
2561                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2562                 }
2563         }
2564 out:
2565         /* if something went wrong then set the changed flag so we try again */
2566         if (retval)
2567                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2568
2569         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2570         return retval;
2571
2572 err_no_memory:
2573         /* Restore elements on the temporary add and delete lists */
2574         spin_lock_bh(&vsi->mac_filter_hash_lock);
2575 err_no_memory_locked:
2576         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2577         i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2578         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2579
2580         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2581         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2582         return -ENOMEM;
2583 }
2584
2585 /**
2586  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2587  * @pf: board private structure
2588  **/
2589 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2590 {
2591         int v;
2592
2593         if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
2594                 return;
2595         pf->flags &= ~I40E_FLAG_FILTER_SYNC;
2596
2597         for (v = 0; v < pf->num_alloc_vsi; v++) {
2598                 if (pf->vsi[v] &&
2599                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) {
2600                         int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2601
2602                         if (ret) {
2603                                 /* come back and try again later */
2604                                 pf->flags |= I40E_FLAG_FILTER_SYNC;
2605                                 break;
2606                         }
2607                 }
2608         }
2609 }
2610
2611 /**
2612  * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2613  * @vsi: the vsi
2614  **/
2615 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
2616 {
2617         if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2618                 return I40E_RXBUFFER_2048;
2619         else
2620                 return I40E_RXBUFFER_3072;
2621 }
2622
2623 /**
2624  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2625  * @netdev: network interface device structure
2626  * @new_mtu: new value for maximum frame size
2627  *
2628  * Returns 0 on success, negative on failure
2629  **/
2630 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2631 {
2632         struct i40e_netdev_priv *np = netdev_priv(netdev);
2633         struct i40e_vsi *vsi = np->vsi;
2634         struct i40e_pf *pf = vsi->back;
2635
2636         if (i40e_enabled_xdp_vsi(vsi)) {
2637                 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2638
2639                 if (frame_size > i40e_max_xdp_frame_size(vsi))
2640                         return -EINVAL;
2641         }
2642
2643         netdev_info(netdev, "changing MTU from %d to %d\n",
2644                     netdev->mtu, new_mtu);
2645         netdev->mtu = new_mtu;
2646         if (netif_running(netdev))
2647                 i40e_vsi_reinit_locked(vsi);
2648         pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED |
2649                       I40E_FLAG_CLIENT_L2_CHANGE);
2650         return 0;
2651 }
2652
2653 /**
2654  * i40e_ioctl - Access the hwtstamp interface
2655  * @netdev: network interface device structure
2656  * @ifr: interface request data
2657  * @cmd: ioctl command
2658  **/
2659 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2660 {
2661         struct i40e_netdev_priv *np = netdev_priv(netdev);
2662         struct i40e_pf *pf = np->vsi->back;
2663
2664         switch (cmd) {
2665         case SIOCGHWTSTAMP:
2666                 return i40e_ptp_get_ts_config(pf, ifr);
2667         case SIOCSHWTSTAMP:
2668                 return i40e_ptp_set_ts_config(pf, ifr);
2669         default:
2670                 return -EOPNOTSUPP;
2671         }
2672 }
2673
2674 /**
2675  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2676  * @vsi: the vsi being adjusted
2677  **/
2678 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2679 {
2680         struct i40e_vsi_context ctxt;
2681         i40e_status ret;
2682
2683         if ((vsi->info.valid_sections &
2684              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2685             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2686                 return;  /* already enabled */
2687
2688         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2689         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2690                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2691
2692         ctxt.seid = vsi->seid;
2693         ctxt.info = vsi->info;
2694         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2695         if (ret) {
2696                 dev_info(&vsi->back->pdev->dev,
2697                          "update vlan stripping failed, err %s aq_err %s\n",
2698                          i40e_stat_str(&vsi->back->hw, ret),
2699                          i40e_aq_str(&vsi->back->hw,
2700                                      vsi->back->hw.aq.asq_last_status));
2701         }
2702 }
2703
2704 /**
2705  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2706  * @vsi: the vsi being adjusted
2707  **/
2708 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2709 {
2710         struct i40e_vsi_context ctxt;
2711         i40e_status ret;
2712
2713         if ((vsi->info.valid_sections &
2714              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2715             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2716              I40E_AQ_VSI_PVLAN_EMOD_MASK))
2717                 return;  /* already disabled */
2718
2719         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2720         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2721                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2722
2723         ctxt.seid = vsi->seid;
2724         ctxt.info = vsi->info;
2725         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2726         if (ret) {
2727                 dev_info(&vsi->back->pdev->dev,
2728                          "update vlan stripping failed, err %s aq_err %s\n",
2729                          i40e_stat_str(&vsi->back->hw, ret),
2730                          i40e_aq_str(&vsi->back->hw,
2731                                      vsi->back->hw.aq.asq_last_status));
2732         }
2733 }
2734
2735 /**
2736  * i40e_vlan_rx_register - Setup or shutdown vlan offload
2737  * @netdev: network interface to be adjusted
2738  * @features: netdev features to test if VLAN offload is enabled or not
2739  **/
2740 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
2741 {
2742         struct i40e_netdev_priv *np = netdev_priv(netdev);
2743         struct i40e_vsi *vsi = np->vsi;
2744
2745         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2746                 i40e_vlan_stripping_enable(vsi);
2747         else
2748                 i40e_vlan_stripping_disable(vsi);
2749 }
2750
2751 /**
2752  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
2753  * @vsi: the vsi being configured
2754  * @vid: vlan id to be added (0 = untagged only , -1 = any)
2755  *
2756  * This is a helper function for adding a new MAC/VLAN filter with the
2757  * specified VLAN for each existing MAC address already in the hash table.
2758  * This function does *not* perform any accounting to update filters based on
2759  * VLAN mode.
2760  *
2761  * NOTE: this function expects to be called while under the
2762  * mac_filter_hash_lock
2763  **/
2764 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2765 {
2766         struct i40e_mac_filter *f, *add_f;
2767         struct hlist_node *h;
2768         int bkt;
2769
2770         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2771                 if (f->state == I40E_FILTER_REMOVE)
2772                         continue;
2773                 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2774                 if (!add_f) {
2775                         dev_info(&vsi->back->pdev->dev,
2776                                  "Could not add vlan filter %d for %pM\n",
2777                                  vid, f->macaddr);
2778                         return -ENOMEM;
2779                 }
2780         }
2781
2782         return 0;
2783 }
2784
2785 /**
2786  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
2787  * @vsi: the VSI being configured
2788  * @vid: VLAN id to be added
2789  **/
2790 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
2791 {
2792         int err;
2793
2794         if (vsi->info.pvid)
2795                 return -EINVAL;
2796
2797         /* The network stack will attempt to add VID=0, with the intention to
2798          * receive priority tagged packets with a VLAN of 0. Our HW receives
2799          * these packets by default when configured to receive untagged
2800          * packets, so we don't need to add a filter for this case.
2801          * Additionally, HW interprets adding a VID=0 filter as meaning to
2802          * receive *only* tagged traffic and stops receiving untagged traffic.
2803          * Thus, we do not want to actually add a filter for VID=0
2804          */
2805         if (!vid)
2806                 return 0;
2807
2808         /* Locked once because all functions invoked below iterates list*/
2809         spin_lock_bh(&vsi->mac_filter_hash_lock);
2810         err = i40e_add_vlan_all_mac(vsi, vid);
2811         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2812         if (err)
2813                 return err;
2814
2815         /* schedule our worker thread which will take care of
2816          * applying the new filter changes
2817          */
2818         i40e_service_event_schedule(vsi->back);
2819         return 0;
2820 }
2821
2822 /**
2823  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
2824  * @vsi: the vsi being configured
2825  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2826  *
2827  * This function should be used to remove all VLAN filters which match the
2828  * given VID. It does not schedule the service event and does not take the
2829  * mac_filter_hash_lock so it may be combined with other operations under
2830  * a single invocation of the mac_filter_hash_lock.
2831  *
2832  * NOTE: this function expects to be called while under the
2833  * mac_filter_hash_lock
2834  */
2835 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2836 {
2837         struct i40e_mac_filter *f;
2838         struct hlist_node *h;
2839         int bkt;
2840
2841         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2842                 if (f->vlan == vid)
2843                         __i40e_del_filter(vsi, f);
2844         }
2845 }
2846
2847 /**
2848  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
2849  * @vsi: the VSI being configured
2850  * @vid: VLAN id to be removed
2851  **/
2852 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
2853 {
2854         if (!vid || vsi->info.pvid)
2855                 return;
2856
2857         spin_lock_bh(&vsi->mac_filter_hash_lock);
2858         i40e_rm_vlan_all_mac(vsi, vid);
2859         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2860
2861         /* schedule our worker thread which will take care of
2862          * applying the new filter changes
2863          */
2864         i40e_service_event_schedule(vsi->back);
2865 }
2866
2867 /**
2868  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2869  * @netdev: network interface to be adjusted
2870  * @vid: vlan id to be added
2871  *
2872  * net_device_ops implementation for adding vlan ids
2873  **/
2874 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2875                                 __always_unused __be16 proto, u16 vid)
2876 {
2877         struct i40e_netdev_priv *np = netdev_priv(netdev);
2878         struct i40e_vsi *vsi = np->vsi;
2879         int ret = 0;
2880
2881         if (vid >= VLAN_N_VID)
2882                 return -EINVAL;
2883
2884         ret = i40e_vsi_add_vlan(vsi, vid);
2885         if (!ret)
2886                 set_bit(vid, vsi->active_vlans);
2887
2888         return ret;
2889 }
2890
2891 /**
2892  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2893  * @netdev: network interface to be adjusted
2894  * @vid: vlan id to be removed
2895  *
2896  * net_device_ops implementation for removing vlan ids
2897  **/
2898 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2899                                  __always_unused __be16 proto, u16 vid)
2900 {
2901         struct i40e_netdev_priv *np = netdev_priv(netdev);
2902         struct i40e_vsi *vsi = np->vsi;
2903
2904         /* return code is ignored as there is nothing a user
2905          * can do about failure to remove and a log message was
2906          * already printed from the other function
2907          */
2908         i40e_vsi_kill_vlan(vsi, vid);
2909
2910         clear_bit(vid, vsi->active_vlans);
2911
2912         return 0;
2913 }
2914
2915 /**
2916  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2917  * @vsi: the vsi being brought back up
2918  **/
2919 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2920 {
2921         u16 vid;
2922
2923         if (!vsi->netdev)
2924                 return;
2925
2926         i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
2927
2928         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2929                 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
2930                                      vid);
2931 }
2932
2933 /**
2934  * i40e_vsi_add_pvid - Add pvid for the VSI
2935  * @vsi: the vsi being adjusted
2936  * @vid: the vlan id to set as a PVID
2937  **/
2938 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2939 {
2940         struct i40e_vsi_context ctxt;
2941         i40e_status ret;
2942
2943         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2944         vsi->info.pvid = cpu_to_le16(vid);
2945         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2946                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2947                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2948
2949         ctxt.seid = vsi->seid;
2950         ctxt.info = vsi->info;
2951         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2952         if (ret) {
2953                 dev_info(&vsi->back->pdev->dev,
2954                          "add pvid failed, err %s aq_err %s\n",
2955                          i40e_stat_str(&vsi->back->hw, ret),
2956                          i40e_aq_str(&vsi->back->hw,
2957                                      vsi->back->hw.aq.asq_last_status));
2958                 return -ENOENT;
2959         }
2960
2961         return 0;
2962 }
2963
2964 /**
2965  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
2966  * @vsi: the vsi being adjusted
2967  *
2968  * Just use the vlan_rx_register() service to put it back to normal
2969  **/
2970 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2971 {
2972         i40e_vlan_stripping_disable(vsi);
2973
2974         vsi->info.pvid = 0;
2975 }
2976
2977 /**
2978  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
2979  * @vsi: ptr to the VSI
2980  *
2981  * If this function returns with an error, then it's possible one or
2982  * more of the rings is populated (while the rest are not).  It is the
2983  * callers duty to clean those orphaned rings.
2984  *
2985  * Return 0 on success, negative on failure
2986  **/
2987 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2988 {
2989         int i, err = 0;
2990
2991         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2992                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2993
2994         if (!i40e_enabled_xdp_vsi(vsi))
2995                 return err;
2996
2997         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2998                 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
2999
3000         return err;
3001 }
3002
3003 /**
3004  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3005  * @vsi: ptr to the VSI
3006  *
3007  * Free VSI's transmit software resources
3008  **/
3009 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3010 {
3011         int i;
3012
3013         if (vsi->tx_rings) {
3014                 for (i = 0; i < vsi->num_queue_pairs; i++)
3015                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3016                                 i40e_free_tx_resources(vsi->tx_rings[i]);
3017         }
3018
3019         if (vsi->xdp_rings) {
3020                 for (i = 0; i < vsi->num_queue_pairs; i++)
3021                         if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3022                                 i40e_free_tx_resources(vsi->xdp_rings[i]);
3023         }
3024 }
3025
3026 /**
3027  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3028  * @vsi: ptr to the VSI
3029  *
3030  * If this function returns with an error, then it's possible one or
3031  * more of the rings is populated (while the rest are not).  It is the
3032  * callers duty to clean those orphaned rings.
3033  *
3034  * Return 0 on success, negative on failure
3035  **/
3036 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3037 {
3038         int i, err = 0;
3039
3040         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3041                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3042         return err;
3043 }
3044
3045 /**
3046  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3047  * @vsi: ptr to the VSI
3048  *
3049  * Free all receive software resources
3050  **/
3051 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3052 {
3053         int i;
3054
3055         if (!vsi->rx_rings)
3056                 return;
3057
3058         for (i = 0; i < vsi->num_queue_pairs; i++)
3059                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3060                         i40e_free_rx_resources(vsi->rx_rings[i]);
3061 }
3062
3063 /**
3064  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3065  * @ring: The Tx ring to configure
3066  *
3067  * This enables/disables XPS for a given Tx descriptor ring
3068  * based on the TCs enabled for the VSI that ring belongs to.
3069  **/
3070 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3071 {
3072         int cpu;
3073
3074         if (!ring->q_vector || !ring->netdev || ring->ch)
3075                 return;
3076
3077         /* We only initialize XPS once, so as not to overwrite user settings */
3078         if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3079                 return;
3080
3081         cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3082         netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3083                             ring->queue_index);
3084 }
3085
3086 /**
3087  * i40e_configure_tx_ring - Configure a transmit ring context and rest
3088  * @ring: The Tx ring to configure
3089  *
3090  * Configure the Tx descriptor ring in the HMC context.
3091  **/
3092 static int i40e_configure_tx_ring(struct i40e_ring *ring)
3093 {
3094         struct i40e_vsi *vsi = ring->vsi;
3095         u16 pf_q = vsi->base_queue + ring->queue_index;
3096         struct i40e_hw *hw = &vsi->back->hw;
3097         struct i40e_hmc_obj_txq tx_ctx;
3098         i40e_status err = 0;
3099         u32 qtx_ctl = 0;
3100
3101         /* some ATR related tx ring init */
3102         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
3103                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
3104                 ring->atr_count = 0;
3105         } else {
3106                 ring->atr_sample_rate = 0;
3107         }
3108
3109         /* configure XPS */
3110         i40e_config_xps_tx_ring(ring);
3111
3112         /* clear the context structure first */
3113         memset(&tx_ctx, 0, sizeof(tx_ctx));
3114
3115         tx_ctx.new_context = 1;
3116         tx_ctx.base = (ring->dma / 128);
3117         tx_ctx.qlen = ring->count;
3118         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
3119                                                I40E_FLAG_FD_ATR_ENABLED));
3120         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
3121         /* FDIR VSI tx ring can still use RS bit and writebacks */
3122         if (vsi->type != I40E_VSI_FDIR)
3123                 tx_ctx.head_wb_ena = 1;
3124         tx_ctx.head_wb_addr = ring->dma +
3125                               (ring->count * sizeof(struct i40e_tx_desc));
3126
3127         /* As part of VSI creation/update, FW allocates certain
3128          * Tx arbitration queue sets for each TC enabled for
3129          * the VSI. The FW returns the handles to these queue
3130          * sets as part of the response buffer to Add VSI,
3131          * Update VSI, etc. AQ commands. It is expected that
3132          * these queue set handles be associated with the Tx
3133          * queues by the driver as part of the TX queue context
3134          * initialization. This has to be done regardless of
3135          * DCB as by default everything is mapped to TC0.
3136          */
3137
3138         if (ring->ch)
3139                 tx_ctx.rdylist =
3140                         le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3141
3142         else
3143                 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3144
3145         tx_ctx.rdylist_act = 0;
3146
3147         /* clear the context in the HMC */
3148         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3149         if (err) {
3150                 dev_info(&vsi->back->pdev->dev,
3151                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3152                          ring->queue_index, pf_q, err);
3153                 return -ENOMEM;
3154         }
3155
3156         /* set the context in the HMC */
3157         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3158         if (err) {
3159                 dev_info(&vsi->back->pdev->dev,
3160                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3161                          ring->queue_index, pf_q, err);
3162                 return -ENOMEM;
3163         }
3164
3165         /* Now associate this queue with this PCI function */
3166         if (ring->ch) {
3167                 if (ring->ch->type == I40E_VSI_VMDQ2)
3168                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3169                 else
3170                         return -EINVAL;
3171
3172                 qtx_ctl |= (ring->ch->vsi_number <<
3173                             I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3174                             I40E_QTX_CTL_VFVM_INDX_MASK;
3175         } else {
3176                 if (vsi->type == I40E_VSI_VMDQ2) {
3177                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3178                         qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3179                                     I40E_QTX_CTL_VFVM_INDX_MASK;
3180                 } else {
3181                         qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3182                 }
3183         }
3184
3185         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3186                     I40E_QTX_CTL_PF_INDX_MASK);
3187         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3188         i40e_flush(hw);
3189
3190         /* cache tail off for easier writes later */
3191         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3192
3193         return 0;
3194 }
3195
3196 /**
3197  * i40e_configure_rx_ring - Configure a receive ring context
3198  * @ring: The Rx ring to configure
3199  *
3200  * Configure the Rx descriptor ring in the HMC context.
3201  **/
3202 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3203 {
3204         struct i40e_vsi *vsi = ring->vsi;
3205         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3206         u16 pf_q = vsi->base_queue + ring->queue_index;
3207         struct i40e_hw *hw = &vsi->back->hw;
3208         struct i40e_hmc_obj_rxq rx_ctx;
3209         i40e_status err = 0;
3210
3211         bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3212
3213         /* clear the context structure first */
3214         memset(&rx_ctx, 0, sizeof(rx_ctx));
3215
3216         ring->rx_buf_len = vsi->rx_buf_len;
3217
3218         rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3219                                     BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3220
3221         rx_ctx.base = (ring->dma / 128);
3222         rx_ctx.qlen = ring->count;
3223
3224         /* use 32 byte descriptors */
3225         rx_ctx.dsize = 1;
3226
3227         /* descriptor type is always zero
3228          * rx_ctx.dtype = 0;
3229          */
3230         rx_ctx.hsplit_0 = 0;
3231
3232         rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3233         if (hw->revision_id == 0)
3234                 rx_ctx.lrxqthresh = 0;
3235         else
3236                 rx_ctx.lrxqthresh = 1;
3237         rx_ctx.crcstrip = 1;
3238         rx_ctx.l2tsel = 1;
3239         /* this controls whether VLAN is stripped from inner headers */
3240         rx_ctx.showiv = 0;
3241         /* set the prefena field to 1 because the manual says to */
3242         rx_ctx.prefena = 1;
3243
3244         /* clear the context in the HMC */
3245         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3246         if (err) {
3247                 dev_info(&vsi->back->pdev->dev,
3248                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3249                          ring->queue_index, pf_q, err);
3250                 return -ENOMEM;
3251         }
3252
3253         /* set the context in the HMC */
3254         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3255         if (err) {
3256                 dev_info(&vsi->back->pdev->dev,
3257                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3258                          ring->queue_index, pf_q, err);
3259                 return -ENOMEM;
3260         }
3261
3262         /* configure Rx buffer alignment */
3263         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3264                 clear_ring_build_skb_enabled(ring);
3265         else
3266                 set_ring_build_skb_enabled(ring);
3267
3268         /* cache tail for quicker writes, and clear the reg before use */
3269         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3270         writel(0, ring->tail);
3271
3272         i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3273
3274         return 0;
3275 }
3276
3277 /**
3278  * i40e_vsi_configure_tx - Configure the VSI for Tx
3279  * @vsi: VSI structure describing this set of rings and resources
3280  *
3281  * Configure the Tx VSI for operation.
3282  **/
3283 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3284 {
3285         int err = 0;
3286         u16 i;
3287
3288         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3289                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3290
3291         if (!i40e_enabled_xdp_vsi(vsi))
3292                 return err;
3293
3294         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3295                 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3296
3297         return err;
3298 }
3299
3300 /**
3301  * i40e_vsi_configure_rx - Configure the VSI for Rx
3302  * @vsi: the VSI being configured
3303  *
3304  * Configure the Rx VSI for operation.
3305  **/
3306 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3307 {
3308         int err = 0;
3309         u16 i;
3310
3311         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3312                 vsi->max_frame = I40E_MAX_RXBUFFER;
3313                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
3314 #if (PAGE_SIZE < 8192)
3315         } else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
3316                    (vsi->netdev->mtu <= ETH_DATA_LEN)) {
3317                 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3318                 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3319 #endif
3320         } else {
3321                 vsi->max_frame = I40E_MAX_RXBUFFER;
3322                 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
3323                                                        I40E_RXBUFFER_2048;
3324         }
3325
3326         /* set up individual rings */
3327         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3328                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3329
3330         return err;
3331 }
3332
3333 /**
3334  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3335  * @vsi: ptr to the VSI
3336  **/
3337 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3338 {
3339         struct i40e_ring *tx_ring, *rx_ring;
3340         u16 qoffset, qcount;
3341         int i, n;
3342
3343         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3344                 /* Reset the TC information */
3345                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3346                         rx_ring = vsi->rx_rings[i];
3347                         tx_ring = vsi->tx_rings[i];
3348                         rx_ring->dcb_tc = 0;
3349                         tx_ring->dcb_tc = 0;
3350                 }
3351                 return;
3352         }
3353
3354         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3355                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3356                         continue;
3357
3358                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3359                 qcount = vsi->tc_config.tc_info[n].qcount;
3360                 for (i = qoffset; i < (qoffset + qcount); i++) {
3361                         rx_ring = vsi->rx_rings[i];
3362                         tx_ring = vsi->tx_rings[i];
3363                         rx_ring->dcb_tc = n;
3364                         tx_ring->dcb_tc = n;
3365                 }
3366         }
3367 }
3368
3369 /**
3370  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3371  * @vsi: ptr to the VSI
3372  **/
3373 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3374 {
3375         if (vsi->netdev)
3376                 i40e_set_rx_mode(vsi->netdev);
3377 }
3378
3379 /**
3380  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3381  * @vsi: Pointer to the targeted VSI
3382  *
3383  * This function replays the hlist on the hw where all the SB Flow Director
3384  * filters were saved.
3385  **/
3386 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3387 {
3388         struct i40e_fdir_filter *filter;
3389         struct i40e_pf *pf = vsi->back;
3390         struct hlist_node *node;
3391
3392         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3393                 return;
3394
3395         /* Reset FDir counters as we're replaying all existing filters */
3396         pf->fd_tcp4_filter_cnt = 0;
3397         pf->fd_udp4_filter_cnt = 0;
3398         pf->fd_sctp4_filter_cnt = 0;
3399         pf->fd_ip4_filter_cnt = 0;
3400
3401         hlist_for_each_entry_safe(filter, node,
3402                                   &pf->fdir_filter_list, fdir_node) {
3403                 i40e_add_del_fdir(vsi, filter, true);
3404         }
3405 }
3406
3407 /**
3408  * i40e_vsi_configure - Set up the VSI for action
3409  * @vsi: the VSI being configured
3410  **/
3411 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3412 {
3413         int err;
3414
3415         i40e_set_vsi_rx_mode(vsi);
3416         i40e_restore_vlan(vsi);
3417         i40e_vsi_config_dcb_rings(vsi);
3418         err = i40e_vsi_configure_tx(vsi);
3419         if (!err)
3420                 err = i40e_vsi_configure_rx(vsi);
3421
3422         return err;
3423 }
3424
3425 /**
3426  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3427  * @vsi: the VSI being configured
3428  **/
3429 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3430 {
3431         bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3432         struct i40e_pf *pf = vsi->back;
3433         struct i40e_hw *hw = &pf->hw;
3434         u16 vector;
3435         int i, q;
3436         u32 qp;
3437
3438         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3439          * and PFINT_LNKLSTn registers, e.g.:
3440          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3441          */
3442         qp = vsi->base_queue;
3443         vector = vsi->base_vector;
3444         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3445                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3446
3447                 q_vector->itr_countdown = ITR_COUNTDOWN_START;
3448                 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting);
3449                 q_vector->rx.latency_range = I40E_LOW_LATENCY;
3450                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3451                      q_vector->rx.itr);
3452                 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting);
3453                 q_vector->tx.latency_range = I40E_LOW_LATENCY;
3454                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3455                      q_vector->tx.itr);
3456                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3457                      i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3458
3459                 /* Linked list for the queuepairs assigned to this vector */
3460                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3461                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3462                         u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3463                         u32 val;
3464
3465                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3466                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3467                               (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3468                               (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3469                               (I40E_QUEUE_TYPE_TX <<
3470                                I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3471
3472                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3473
3474                         if (has_xdp) {
3475                                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3476                                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3477                                       (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3478                                       (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3479                                       (I40E_QUEUE_TYPE_TX <<
3480                                        I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3481
3482                                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3483                         }
3484
3485                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3486                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3487                               (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3488                               ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3489                               (I40E_QUEUE_TYPE_RX <<
3490                                I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3491
3492                         /* Terminate the linked list */
3493                         if (q == (q_vector->num_ringpairs - 1))
3494                                 val |= (I40E_QUEUE_END_OF_LIST <<
3495                                         I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3496
3497                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3498                         qp++;
3499                 }
3500         }
3501
3502         i40e_flush(hw);
3503 }
3504
3505 /**
3506  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3507  * @hw: ptr to the hardware info
3508  **/
3509 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3510 {
3511         struct i40e_hw *hw = &pf->hw;
3512         u32 val;
3513
3514         /* clear things first */
3515         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3516         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3517
3518         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3519               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3520               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3521               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3522               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3523               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3524               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3525               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3526
3527         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3528                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3529
3530         if (pf->flags & I40E_FLAG_PTP)
3531                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3532
3533         wr32(hw, I40E_PFINT_ICR0_ENA, val);
3534
3535         /* SW_ITR_IDX = 0, but don't change INTENA */
3536         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3537                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3538
3539         /* OTHER_ITR_IDX = 0 */
3540         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3541 }
3542
3543 /**
3544  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3545  * @vsi: the VSI being configured
3546  **/
3547 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3548 {
3549         u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3550         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3551         struct i40e_pf *pf = vsi->back;
3552         struct i40e_hw *hw = &pf->hw;
3553         u32 val;
3554
3555         /* set the ITR configuration */
3556         q_vector->itr_countdown = ITR_COUNTDOWN_START;
3557         q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting);
3558         q_vector->rx.latency_range = I40E_LOW_LATENCY;
3559         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
3560         q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting);
3561         q_vector->tx.latency_range = I40E_LOW_LATENCY;
3562         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
3563
3564         i40e_enable_misc_int_causes(pf);
3565
3566         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3567         wr32(hw, I40E_PFINT_LNKLST0, 0);
3568
3569         /* Associate the queue pair to the vector and enable the queue int */
3570         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                   |
3571               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
3572               (nextqp      << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3573               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3574
3575         wr32(hw, I40E_QINT_RQCTL(0), val);
3576
3577         if (i40e_enabled_xdp_vsi(vsi)) {
3578                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                 |
3579                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
3580                       (I40E_QUEUE_TYPE_TX
3581                        << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3582
3583                wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3584         }
3585
3586         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
3587               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3588               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3589
3590         wr32(hw, I40E_QINT_TQCTL(0), val);
3591         i40e_flush(hw);
3592 }
3593
3594 /**
3595  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3596  * @pf: board private structure
3597  **/
3598 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3599 {
3600         struct i40e_hw *hw = &pf->hw;
3601
3602         wr32(hw, I40E_PFINT_DYN_CTL0,
3603              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3604         i40e_flush(hw);
3605 }
3606
3607 /**
3608  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3609  * @pf: board private structure
3610  **/
3611 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
3612 {
3613         struct i40e_hw *hw = &pf->hw;
3614         u32 val;
3615
3616         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
3617               I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3618               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3619
3620         wr32(hw, I40E_PFINT_DYN_CTL0, val);
3621         i40e_flush(hw);
3622 }
3623
3624 /**
3625  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3626  * @irq: interrupt number
3627  * @data: pointer to a q_vector
3628  **/
3629 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3630 {
3631         struct i40e_q_vector *q_vector = data;
3632
3633         if (!q_vector->tx.ring && !q_vector->rx.ring)
3634                 return IRQ_HANDLED;
3635
3636         napi_schedule_irqoff(&q_vector->napi);
3637
3638         return IRQ_HANDLED;
3639 }
3640
3641 /**
3642  * i40e_irq_affinity_notify - Callback for affinity changes
3643  * @notify: context as to what irq was changed
3644  * @mask: the new affinity mask
3645  *
3646  * This is a callback function used by the irq_set_affinity_notifier function
3647  * so that we may register to receive changes to the irq affinity masks.
3648  **/
3649 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3650                                      const cpumask_t *mask)
3651 {
3652         struct i40e_q_vector *q_vector =
3653                 container_of(notify, struct i40e_q_vector, affinity_notify);
3654
3655         cpumask_copy(&q_vector->affinity_mask, mask);
3656 }
3657
3658 /**
3659  * i40e_irq_affinity_release - Callback for affinity notifier release
3660  * @ref: internal core kernel usage
3661  *
3662  * This is a callback function used by the irq_set_affinity_notifier function
3663  * to inform the current notification subscriber that they will no longer
3664  * receive notifications.
3665  **/
3666 static void i40e_irq_affinity_release(struct kref *ref) {}
3667
3668 /**
3669  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3670  * @vsi: the VSI being configured
3671  * @basename: name for the vector
3672  *
3673  * Allocates MSI-X vectors and requests interrupts from the kernel.
3674  **/
3675 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3676 {
3677         int q_vectors = vsi->num_q_vectors;
3678         struct i40e_pf *pf = vsi->back;
3679         int base = vsi->base_vector;
3680         int rx_int_idx = 0;
3681         int tx_int_idx = 0;
3682         int vector, err;
3683         int irq_num;
3684         int cpu;
3685
3686         for (vector = 0; vector < q_vectors; vector++) {
3687                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3688
3689                 irq_num = pf->msix_entries[base + vector].vector;
3690
3691                 if (q_vector->tx.ring && q_vector->rx.ring) {
3692                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3693                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3694                         tx_int_idx++;
3695                 } else if (q_vector->rx.ring) {
3696                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3697                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
3698                 } else if (q_vector->tx.ring) {
3699                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3700                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
3701                 } else {
3702                         /* skip this unused q_vector */
3703                         continue;
3704                 }
3705                 err = request_irq(irq_num,
3706                                   vsi->irq_handler,
3707                                   0,
3708                                   q_vector->name,
3709                                   q_vector);
3710                 if (err) {
3711                         dev_info(&pf->pdev->dev,
3712                                  "MSIX request_irq failed, error: %d\n", err);
3713                         goto free_queue_irqs;
3714                 }
3715
3716                 /* register for affinity change notifications */
3717                 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3718                 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3719                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3720                 /* Spread affinity hints out across online CPUs.
3721                  *
3722                  * get_cpu_mask returns a static constant mask with
3723                  * a permanent lifetime so it's ok to pass to
3724                  * irq_set_affinity_hint without making a copy.
3725                  */
3726                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
3727                 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
3728         }
3729
3730         vsi->irqs_ready = true;
3731         return 0;
3732
3733 free_queue_irqs:
3734         while (vector) {
3735                 vector--;
3736                 irq_num = pf->msix_entries[base + vector].vector;
3737                 irq_set_affinity_notifier(irq_num, NULL);
3738                 irq_set_affinity_hint(irq_num, NULL);
3739                 free_irq(irq_num, &vsi->q_vectors[vector]);
3740         }
3741         return err;
3742 }
3743
3744 /**
3745  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3746  * @vsi: the VSI being un-configured
3747  **/
3748 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3749 {
3750         struct i40e_pf *pf = vsi->back;
3751         struct i40e_hw *hw = &pf->hw;
3752         int base = vsi->base_vector;
3753         int i;
3754
3755         /* disable interrupt causation from each queue */
3756         for (i = 0; i < vsi->num_queue_pairs; i++) {
3757                 u32 val;
3758
3759                 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
3760                 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
3761                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
3762
3763                 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
3764                 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
3765                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
3766
3767                 if (!i40e_enabled_xdp_vsi(vsi))
3768                         continue;
3769                 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
3770         }
3771
3772         /* disable each interrupt */
3773         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3774                 for (i = vsi->base_vector;
3775                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3776                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3777
3778                 i40e_flush(hw);
3779                 for (i = 0; i < vsi->num_q_vectors; i++)
3780                         synchronize_irq(pf->msix_entries[i + base].vector);
3781         } else {
3782                 /* Legacy and MSI mode - this stops all interrupt handling */
3783                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3784                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3785                 i40e_flush(hw);
3786                 synchronize_irq(pf->pdev->irq);
3787         }
3788 }
3789
3790 /**
3791  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3792  * @vsi: the VSI being configured
3793  **/
3794 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3795 {
3796         struct i40e_pf *pf = vsi->back;
3797         int i;
3798
3799         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3800                 for (i = 0; i < vsi->num_q_vectors; i++)
3801                         i40e_irq_dynamic_enable(vsi, i);
3802         } else {
3803                 i40e_irq_dynamic_enable_icr0(pf);
3804         }
3805
3806         i40e_flush(&pf->hw);
3807         return 0;
3808 }
3809
3810 /**
3811  * i40e_free_misc_vector - Free the vector that handles non-queue events
3812  * @pf: board private structure
3813  **/
3814 static void i40e_free_misc_vector(struct i40e_pf *pf)
3815 {
3816         /* Disable ICR 0 */
3817         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3818         i40e_flush(&pf->hw);
3819
3820         if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
3821                 synchronize_irq(pf->msix_entries[0].vector);
3822                 free_irq(pf->msix_entries[0].vector, pf);
3823                 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
3824         }
3825 }
3826
3827 /**
3828  * i40e_intr - MSI/Legacy and non-queue interrupt handler
3829  * @irq: interrupt number
3830  * @data: pointer to a q_vector
3831  *
3832  * This is the handler used for all MSI/Legacy interrupts, and deals
3833  * with both queue and non-queue interrupts.  This is also used in
3834  * MSIX mode to handle the non-queue interrupts.
3835  **/
3836 static irqreturn_t i40e_intr(int irq, void *data)
3837 {
3838         struct i40e_pf *pf = (struct i40e_pf *)data;
3839         struct i40e_hw *hw = &pf->hw;
3840         irqreturn_t ret = IRQ_NONE;
3841         u32 icr0, icr0_remaining;
3842         u32 val, ena_mask;
3843
3844         icr0 = rd32(hw, I40E_PFINT_ICR0);
3845         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
3846
3847         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
3848         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
3849                 goto enable_intr;
3850
3851         /* if interrupt but no bits showing, must be SWINT */
3852         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
3853             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
3854                 pf->sw_int_count++;
3855
3856         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
3857             (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
3858                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3859                 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
3860                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
3861         }
3862
3863         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
3864         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
3865                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
3866                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3867
3868                 /* We do not have a way to disarm Queue causes while leaving
3869                  * interrupt enabled for all other causes, ideally
3870                  * interrupt should be disabled while we are in NAPI but
3871                  * this is not a performance path and napi_schedule()
3872                  * can deal with rescheduling.
3873                  */
3874                 if (!test_bit(__I40E_DOWN, pf->state))
3875                         napi_schedule_irqoff(&q_vector->napi);
3876         }
3877
3878         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3879                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3880                 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
3881                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
3882         }
3883
3884         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
3885                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3886                 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
3887         }
3888
3889         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
3890                 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
3891                 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
3892         }
3893
3894         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
3895                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
3896                         set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
3897                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
3898                 val = rd32(hw, I40E_GLGEN_RSTAT);
3899                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
3900                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
3901                 if (val == I40E_RESET_CORER) {
3902                         pf->corer_count++;
3903                 } else if (val == I40E_RESET_GLOBR) {
3904                         pf->globr_count++;
3905                 } else if (val == I40E_RESET_EMPR) {
3906                         pf->empr_count++;
3907                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
3908                 }
3909         }
3910
3911         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
3912                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
3913                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
3914                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
3915                          rd32(hw, I40E_PFHMC_ERRORINFO),
3916                          rd32(hw, I40E_PFHMC_ERRORDATA));
3917         }
3918
3919         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
3920                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
3921
3922                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
3923                         icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3924                         i40e_ptp_tx_hwtstamp(pf);
3925                 }
3926         }
3927
3928         /* If a critical error is pending we have no choice but to reset the
3929          * device.
3930          * Report and mask out any remaining unexpected interrupts.
3931          */
3932         icr0_remaining = icr0 & ena_mask;
3933         if (icr0_remaining) {
3934                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
3935                          icr0_remaining);
3936                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
3937                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
3938                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
3939                         dev_info(&pf->pdev->dev, "device will be reset\n");
3940                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
3941                         i40e_service_event_schedule(pf);
3942                 }
3943                 ena_mask &= ~icr0_remaining;
3944         }
3945         ret = IRQ_HANDLED;
3946
3947 enable_intr:
3948         /* re-enable interrupt causes */
3949         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
3950         if (!test_bit(__I40E_DOWN, pf->state)) {
3951                 i40e_service_event_schedule(pf);
3952                 i40e_irq_dynamic_enable_icr0(pf);
3953         }
3954
3955         return ret;
3956 }
3957
3958 /**
3959  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
3960  * @tx_ring:  tx ring to clean
3961  * @budget:   how many cleans we're allowed
3962  *
3963  * Returns true if there's any budget left (e.g. the clean is finished)
3964  **/
3965 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
3966 {
3967         struct i40e_vsi *vsi = tx_ring->vsi;
3968         u16 i = tx_ring->next_to_clean;
3969         struct i40e_tx_buffer *tx_buf;
3970         struct i40e_tx_desc *tx_desc;
3971
3972         tx_buf = &tx_ring->tx_bi[i];
3973         tx_desc = I40E_TX_DESC(tx_ring, i);
3974         i -= tx_ring->count;
3975
3976         do {
3977                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
3978
3979                 /* if next_to_watch is not set then there is no work pending */
3980                 if (!eop_desc)
3981                         break;
3982
3983                 /* prevent any other reads prior to eop_desc */
3984                 smp_rmb();
3985
3986                 /* if the descriptor isn't done, no work yet to do */
3987                 if (!(eop_desc->cmd_type_offset_bsz &
3988                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
3989                         break;
3990
3991                 /* clear next_to_watch to prevent false hangs */
3992                 tx_buf->next_to_watch = NULL;
3993
3994                 tx_desc->buffer_addr = 0;
3995                 tx_desc->cmd_type_offset_bsz = 0;
3996                 /* move past filter desc */
3997                 tx_buf++;
3998                 tx_desc++;
3999                 i++;
4000                 if (unlikely(!i)) {
4001                         i -= tx_ring->count;
4002                         tx_buf = tx_ring->tx_bi;
4003                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4004                 }
4005                 /* unmap skb header data */
4006                 dma_unmap_single(tx_ring->dev,
4007                                  dma_unmap_addr(tx_buf, dma),
4008                                  dma_unmap_len(tx_buf, len),
4009                                  DMA_TO_DEVICE);
4010                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4011                         kfree(tx_buf->raw_buf);
4012
4013                 tx_buf->raw_buf = NULL;
4014                 tx_buf->tx_flags = 0;
4015                 tx_buf->next_to_watch = NULL;
4016                 dma_unmap_len_set(tx_buf, len, 0);
4017                 tx_desc->buffer_addr = 0;
4018                 tx_desc->cmd_type_offset_bsz = 0;
4019
4020                 /* move us past the eop_desc for start of next FD desc */
4021                 tx_buf++;
4022                 tx_desc++;
4023                 i++;
4024                 if (unlikely(!i)) {
4025                         i -= tx_ring->count;
4026                         tx_buf = tx_ring->tx_bi;
4027                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4028                 }
4029
4030                 /* update budget accounting */
4031                 budget--;
4032         } while (likely(budget));
4033
4034         i += tx_ring->count;
4035         tx_ring->next_to_clean = i;
4036
4037         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
4038                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4039
4040         return budget > 0;
4041 }
4042
4043 /**
4044  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4045  * @irq: interrupt number
4046  * @data: pointer to a q_vector
4047  **/
4048 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4049 {
4050         struct i40e_q_vector *q_vector = data;
4051         struct i40e_vsi *vsi;
4052
4053         if (!q_vector->tx.ring)
4054                 return IRQ_HANDLED;
4055
4056         vsi = q_vector->tx.ring->vsi;
4057         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4058
4059         return IRQ_HANDLED;
4060 }
4061
4062 /**
4063  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4064  * @vsi: the VSI being configured
4065  * @v_idx: vector index
4066  * @qp_idx: queue pair index
4067  **/
4068 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4069 {
4070         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4071         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4072         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4073
4074         tx_ring->q_vector = q_vector;
4075         tx_ring->next = q_vector->tx.ring;
4076         q_vector->tx.ring = tx_ring;
4077         q_vector->tx.count++;
4078
4079         /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4080         if (i40e_enabled_xdp_vsi(vsi)) {
4081                 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4082
4083                 xdp_ring->q_vector = q_vector;
4084                 xdp_ring->next = q_vector->tx.ring;
4085                 q_vector->tx.ring = xdp_ring;
4086                 q_vector->tx.count++;
4087         }
4088
4089         rx_ring->q_vector = q_vector;
4090         rx_ring->next = q_vector->rx.ring;
4091         q_vector->rx.ring = rx_ring;
4092         q_vector->rx.count++;
4093 }
4094
4095 /**
4096  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4097  * @vsi: the VSI being configured
4098  *
4099  * This function maps descriptor rings to the queue-specific vectors
4100  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
4101  * one vector per queue pair, but on a constrained vector budget, we
4102  * group the queue pairs as "efficiently" as possible.
4103  **/
4104 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4105 {
4106         int qp_remaining = vsi->num_queue_pairs;
4107         int q_vectors = vsi->num_q_vectors;
4108         int num_ringpairs;
4109         int v_start = 0;
4110         int qp_idx = 0;
4111
4112         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4113          * group them so there are multiple queues per vector.
4114          * It is also important to go through all the vectors available to be
4115          * sure that if we don't use all the vectors, that the remaining vectors
4116          * are cleared. This is especially important when decreasing the
4117          * number of queues in use.
4118          */
4119         for (; v_start < q_vectors; v_start++) {
4120                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4121
4122                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4123
4124                 q_vector->num_ringpairs = num_ringpairs;
4125
4126                 q_vector->rx.count = 0;
4127                 q_vector->tx.count = 0;
4128                 q_vector->rx.ring = NULL;
4129                 q_vector->tx.ring = NULL;
4130
4131                 while (num_ringpairs--) {
4132                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4133                         qp_idx++;
4134                         qp_remaining--;
4135                 }
4136         }
4137 }
4138
4139 /**
4140  * i40e_vsi_request_irq - Request IRQ from the OS
4141  * @vsi: the VSI being configured
4142  * @basename: name for the vector
4143  **/
4144 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4145 {
4146         struct i40e_pf *pf = vsi->back;
4147         int err;
4148
4149         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4150                 err = i40e_vsi_request_irq_msix(vsi, basename);
4151         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
4152                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
4153                                   pf->int_name, pf);
4154         else
4155                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4156                                   pf->int_name, pf);
4157
4158         if (err)
4159                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4160
4161         return err;
4162 }
4163
4164 #ifdef CONFIG_NET_POLL_CONTROLLER
4165 /**
4166  * i40e_netpoll - A Polling 'interrupt' handler
4167  * @netdev: network interface device structure
4168  *
4169  * This is used by netconsole to send skbs without having to re-enable
4170  * interrupts.  It's not called while the normal interrupt routine is executing.
4171  **/
4172 static void i40e_netpoll(struct net_device *netdev)
4173 {
4174         struct i40e_netdev_priv *np = netdev_priv(netdev);
4175         struct i40e_vsi *vsi = np->vsi;
4176         struct i40e_pf *pf = vsi->back;
4177         int i;
4178
4179         /* if interface is down do nothing */
4180         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4181                 return;
4182
4183         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4184                 for (i = 0; i < vsi->num_q_vectors; i++)
4185                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4186         } else {
4187                 i40e_intr(pf->pdev->irq, netdev);
4188         }
4189 }
4190 #endif
4191
4192 #define I40E_QTX_ENA_WAIT_COUNT 50
4193
4194 /**
4195  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4196  * @pf: the PF being configured
4197  * @pf_q: the PF queue
4198  * @enable: enable or disable state of the queue
4199  *
4200  * This routine will wait for the given Tx queue of the PF to reach the
4201  * enabled or disabled state.
4202  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4203  * multiple retries; else will return 0 in case of success.
4204  **/
4205 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4206 {
4207         int i;
4208         u32 tx_reg;
4209
4210         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4211                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4212                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4213                         break;
4214
4215                 usleep_range(10, 20);
4216         }
4217         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4218                 return -ETIMEDOUT;
4219
4220         return 0;
4221 }
4222
4223 /**
4224  * i40e_control_tx_q - Start or stop a particular Tx queue
4225  * @pf: the PF structure
4226  * @pf_q: the PF queue to configure
4227  * @enable: start or stop the queue
4228  *
4229  * This function enables or disables a single queue. Note that any delay
4230  * required after the operation is expected to be handled by the caller of
4231  * this function.
4232  **/
4233 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4234 {
4235         struct i40e_hw *hw = &pf->hw;
4236         u32 tx_reg;
4237         int i;
4238
4239         /* warn the TX unit of coming changes */
4240         i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4241         if (!enable)
4242                 usleep_range(10, 20);
4243
4244         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4245                 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4246                 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4247                     ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4248                         break;
4249                 usleep_range(1000, 2000);
4250         }
4251
4252         /* Skip if the queue is already in the requested state */
4253         if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4254                 return;
4255
4256         /* turn on/off the queue */
4257         if (enable) {
4258                 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4259                 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4260         } else {
4261                 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4262         }
4263
4264         wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4265 }
4266
4267 /**
4268  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4269  * @seid: VSI SEID
4270  * @pf: the PF structure
4271  * @pf_q: the PF queue to configure
4272  * @is_xdp: true if the queue is used for XDP
4273  * @enable: start or stop the queue
4274  **/
4275 static int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4276                                   bool is_xdp, bool enable)
4277 {
4278         int ret;
4279
4280         i40e_control_tx_q(pf, pf_q, enable);
4281
4282         /* wait for the change to finish */
4283         ret = i40e_pf_txq_wait(pf, pf_q, enable);
4284         if (ret) {
4285                 dev_info(&pf->pdev->dev,
4286                          "VSI seid %d %sTx ring %d %sable timeout\n",
4287                          seid, (is_xdp ? "XDP " : ""), pf_q,
4288                          (enable ? "en" : "dis"));
4289         }
4290
4291         return ret;
4292 }
4293
4294 /**
4295  * i40e_vsi_control_tx - Start or stop a VSI's rings
4296  * @vsi: the VSI being configured
4297  * @enable: start or stop the rings
4298  **/
4299 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
4300 {
4301         struct i40e_pf *pf = vsi->back;
4302         int i, pf_q, ret = 0;
4303
4304         pf_q = vsi->base_queue;
4305         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4306                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4307                                              pf_q,
4308                                              false /*is xdp*/, enable);
4309                 if (ret)
4310                         break;
4311
4312                 if (!i40e_enabled_xdp_vsi(vsi))
4313                         continue;
4314
4315                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4316                                              pf_q + vsi->alloc_queue_pairs,
4317                                              true /*is xdp*/, enable);
4318                 if (ret)
4319                         break;
4320         }
4321
4322         return ret;
4323 }
4324
4325 /**
4326  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4327  * @pf: the PF being configured
4328  * @pf_q: the PF queue
4329  * @enable: enable or disable state of the queue
4330  *
4331  * This routine will wait for the given Rx queue of the PF to reach the
4332  * enabled or disabled state.
4333  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4334  * multiple retries; else will return 0 in case of success.
4335  **/
4336 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4337 {
4338         int i;
4339         u32 rx_reg;
4340
4341         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4342                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4343                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4344                         break;
4345
4346                 usleep_range(10, 20);
4347         }
4348         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4349                 return -ETIMEDOUT;
4350
4351         return 0;
4352 }
4353
4354 /**
4355  * i40e_control_rx_q - Start or stop a particular Rx queue
4356  * @pf: the PF structure
4357  * @pf_q: the PF queue to configure
4358  * @enable: start or stop the queue
4359  *
4360  * This function enables or disables a single queue. Note that any delay
4361  * required after the operation is expected to be handled by the caller of
4362  * this function.
4363  **/
4364 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4365 {
4366         struct i40e_hw *hw = &pf->hw;
4367         u32 rx_reg;
4368         int i;
4369
4370         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4371                 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4372                 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4373                     ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4374                         break;
4375                 usleep_range(1000, 2000);
4376         }
4377
4378         /* Skip if the queue is already in the requested state */
4379         if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4380                 return;
4381
4382         /* turn on/off the queue */
4383         if (enable)
4384                 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4385         else
4386                 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4387
4388         wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4389 }
4390
4391 /**
4392  * i40e_vsi_control_rx - Start or stop a VSI's rings
4393  * @vsi: the VSI being configured
4394  * @enable: start or stop the rings
4395  **/
4396 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
4397 {
4398         struct i40e_pf *pf = vsi->back;
4399         int i, pf_q, ret = 0;
4400
4401         pf_q = vsi->base_queue;
4402         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4403                 i40e_control_rx_q(pf, pf_q, enable);
4404
4405                 /* wait for the change to finish */
4406                 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4407                 if (ret) {
4408                         dev_info(&pf->pdev->dev,
4409                                  "VSI seid %d Rx ring %d %sable timeout\n",
4410                                  vsi->seid, pf_q, (enable ? "en" : "dis"));
4411                         break;
4412                 }
4413         }
4414
4415         /* Due to HW errata, on Rx disable only, the register can indicate done
4416          * before it really is. Needs 50ms to be sure
4417          */
4418         if (!enable)
4419                 mdelay(50);
4420
4421         return ret;
4422 }
4423
4424 /**
4425  * i40e_vsi_start_rings - Start a VSI's rings
4426  * @vsi: the VSI being configured
4427  **/
4428 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4429 {
4430         int ret = 0;
4431
4432         /* do rx first for enable and last for disable */
4433         ret = i40e_vsi_control_rx(vsi, true);
4434         if (ret)
4435                 return ret;
4436         ret = i40e_vsi_control_tx(vsi, true);
4437
4438         return ret;
4439 }
4440
4441 /**
4442  * i40e_vsi_stop_rings - Stop a VSI's rings
4443  * @vsi: the VSI being configured
4444  **/
4445 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4446 {
4447         /* When port TX is suspended, don't wait */
4448         if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4449                 return i40e_vsi_stop_rings_no_wait(vsi);
4450
4451         /* do rx first for enable and last for disable
4452          * Ignore return value, we need to shutdown whatever we can
4453          */
4454         i40e_vsi_control_tx(vsi, false);
4455         i40e_vsi_control_rx(vsi, false);
4456 }
4457
4458 /**
4459  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4460  * @vsi: the VSI being shutdown
4461  *
4462  * This function stops all the rings for a VSI but does not delay to verify
4463  * that rings have been disabled. It is expected that the caller is shutting
4464  * down multiple VSIs at once and will delay together for all the VSIs after
4465  * initiating the shutdown. This is particularly useful for shutting down lots
4466  * of VFs together. Otherwise, a large delay can be incurred while configuring
4467  * each VSI in serial.
4468  **/
4469 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4470 {
4471         struct i40e_pf *pf = vsi->back;
4472         int i, pf_q;
4473
4474         pf_q = vsi->base_queue;
4475         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4476                 i40e_control_tx_q(pf, pf_q, false);
4477                 i40e_control_rx_q(pf, pf_q, false);
4478         }
4479 }
4480
4481 /**
4482  * i40e_vsi_free_irq - Free the irq association with the OS
4483  * @vsi: the VSI being configured
4484  **/
4485 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4486 {
4487         struct i40e_pf *pf = vsi->back;
4488         struct i40e_hw *hw = &pf->hw;
4489         int base = vsi->base_vector;
4490         u32 val, qp;
4491         int i;
4492
4493         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4494                 if (!vsi->q_vectors)
4495                         return;
4496
4497                 if (!vsi->irqs_ready)
4498                         return;
4499
4500                 vsi->irqs_ready = false;
4501                 for (i = 0; i < vsi->num_q_vectors; i++) {
4502                         int irq_num;
4503                         u16 vector;
4504
4505                         vector = i + base;
4506                         irq_num = pf->msix_entries[vector].vector;
4507
4508                         /* free only the irqs that were actually requested */
4509                         if (!vsi->q_vectors[i] ||
4510                             !vsi->q_vectors[i]->num_ringpairs)
4511                                 continue;
4512
4513                         /* clear the affinity notifier in the IRQ descriptor */
4514                         irq_set_affinity_notifier(irq_num, NULL);
4515                         /* remove our suggested affinity mask for this IRQ */
4516                         irq_set_affinity_hint(irq_num, NULL);
4517                         synchronize_irq(irq_num);
4518                         free_irq(irq_num, vsi->q_vectors[i]);
4519
4520                         /* Tear down the interrupt queue link list
4521                          *
4522                          * We know that they come in pairs and always
4523                          * the Rx first, then the Tx.  To clear the
4524                          * link list, stick the EOL value into the
4525                          * next_q field of the registers.
4526                          */
4527                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4528                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4529                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4530                         val |= I40E_QUEUE_END_OF_LIST
4531                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4532                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4533
4534                         while (qp != I40E_QUEUE_END_OF_LIST) {
4535                                 u32 next;
4536
4537                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4538
4539                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4540                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4541                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4542                                          I40E_QINT_RQCTL_INTEVENT_MASK);
4543
4544                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4545                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4546
4547                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4548
4549                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4550
4551                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4552                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4553
4554                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4555                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4556                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4557                                          I40E_QINT_TQCTL_INTEVENT_MASK);
4558
4559                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4560                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4561
4562                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4563                                 qp = next;
4564                         }
4565                 }
4566         } else {
4567                 free_irq(pf->pdev->irq, pf);
4568
4569                 val = rd32(hw, I40E_PFINT_LNKLST0);
4570                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4571                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4572                 val |= I40E_QUEUE_END_OF_LIST
4573                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4574                 wr32(hw, I40E_PFINT_LNKLST0, val);
4575
4576                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4577                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4578                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4579                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4580                          I40E_QINT_RQCTL_INTEVENT_MASK);
4581
4582                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4583                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4584
4585                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4586
4587                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4588
4589                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4590                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4591                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4592                          I40E_QINT_TQCTL_INTEVENT_MASK);
4593
4594                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4595                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4596
4597                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4598         }
4599 }
4600
4601 /**
4602  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4603  * @vsi: the VSI being configured
4604  * @v_idx: Index of vector to be freed
4605  *
4606  * This function frees the memory allocated to the q_vector.  In addition if
4607  * NAPI is enabled it will delete any references to the NAPI struct prior
4608  * to freeing the q_vector.
4609  **/
4610 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4611 {
4612         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4613         struct i40e_ring *ring;
4614
4615         if (!q_vector)
4616                 return;
4617
4618         /* disassociate q_vector from rings */
4619         i40e_for_each_ring(ring, q_vector->tx)
4620                 ring->q_vector = NULL;
4621
4622         i40e_for_each_ring(ring, q_vector->rx)
4623                 ring->q_vector = NULL;
4624
4625         /* only VSI w/ an associated netdev is set up w/ NAPI */
4626         if (vsi->netdev)
4627                 netif_napi_del(&q_vector->napi);
4628
4629         vsi->q_vectors[v_idx] = NULL;
4630
4631         kfree_rcu(q_vector, rcu);
4632 }
4633
4634 /**
4635  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4636  * @vsi: the VSI being un-configured
4637  *
4638  * This frees the memory allocated to the q_vectors and
4639  * deletes references to the NAPI struct.
4640  **/
4641 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4642 {
4643         int v_idx;
4644
4645         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4646                 i40e_free_q_vector(vsi, v_idx);
4647 }
4648
4649 /**
4650  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4651  * @pf: board private structure
4652  **/
4653 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4654 {
4655         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4656         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4657                 pci_disable_msix(pf->pdev);
4658                 kfree(pf->msix_entries);
4659                 pf->msix_entries = NULL;
4660                 kfree(pf->irq_pile);
4661                 pf->irq_pile = NULL;
4662         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4663                 pci_disable_msi(pf->pdev);
4664         }
4665         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4666 }
4667
4668 /**
4669  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4670  * @pf: board private structure
4671  *
4672  * We go through and clear interrupt specific resources and reset the structure
4673  * to pre-load conditions
4674  **/
4675 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4676 {
4677         int i;
4678
4679         i40e_free_misc_vector(pf);
4680
4681         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4682                       I40E_IWARP_IRQ_PILE_ID);
4683
4684         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4685         for (i = 0; i < pf->num_alloc_vsi; i++)
4686                 if (pf->vsi[i])
4687                         i40e_vsi_free_q_vectors(pf->vsi[i]);
4688         i40e_reset_interrupt_capability(pf);
4689 }
4690
4691 /**
4692  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4693  * @vsi: the VSI being configured
4694  **/
4695 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4696 {
4697         int q_idx;
4698
4699         if (!vsi->netdev)
4700                 return;
4701
4702         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4703                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4704
4705                 if (q_vector->rx.ring || q_vector->tx.ring)
4706                         napi_enable(&q_vector->napi);
4707         }
4708 }
4709
4710 /**
4711  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4712  * @vsi: the VSI being configured
4713  **/
4714 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4715 {
4716         int q_idx;
4717
4718         if (!vsi->netdev)
4719                 return;
4720
4721         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4722                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4723
4724                 if (q_vector->rx.ring || q_vector->tx.ring)
4725                         napi_disable(&q_vector->napi);
4726         }
4727 }
4728
4729 /**
4730  * i40e_vsi_close - Shut down a VSI
4731  * @vsi: the vsi to be quelled
4732  **/
4733 static void i40e_vsi_close(struct i40e_vsi *vsi)
4734 {
4735         struct i40e_pf *pf = vsi->back;
4736         if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
4737                 i40e_down(vsi);
4738         i40e_vsi_free_irq(vsi);
4739         i40e_vsi_free_tx_resources(vsi);
4740         i40e_vsi_free_rx_resources(vsi);
4741         vsi->current_netdev_flags = 0;
4742         pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
4743         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4744                 pf->flags |=  I40E_FLAG_CLIENT_RESET;
4745 }
4746
4747 /**
4748  * i40e_quiesce_vsi - Pause a given VSI
4749  * @vsi: the VSI being paused
4750  **/
4751 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4752 {
4753         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4754                 return;
4755
4756         set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
4757         if (vsi->netdev && netif_running(vsi->netdev))
4758                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4759         else
4760                 i40e_vsi_close(vsi);
4761 }
4762
4763 /**
4764  * i40e_unquiesce_vsi - Resume a given VSI
4765  * @vsi: the VSI being resumed
4766  **/
4767 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
4768 {
4769         if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
4770                 return;
4771
4772         if (vsi->netdev && netif_running(vsi->netdev))
4773                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
4774         else
4775                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
4776 }
4777
4778 /**
4779  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
4780  * @pf: the PF
4781  **/
4782 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
4783 {
4784         int v;
4785
4786         for (v = 0; v < pf->num_alloc_vsi; v++) {
4787                 if (pf->vsi[v])
4788                         i40e_quiesce_vsi(pf->vsi[v]);
4789         }
4790 }
4791
4792 /**
4793  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
4794  * @pf: the PF
4795  **/
4796 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
4797 {
4798         int v;
4799
4800         for (v = 0; v < pf->num_alloc_vsi; v++) {
4801                 if (pf->vsi[v])
4802                         i40e_unquiesce_vsi(pf->vsi[v]);
4803         }
4804 }
4805
4806 /**
4807  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
4808  * @vsi: the VSI being configured
4809  *
4810  * Wait until all queues on a given VSI have been disabled.
4811  **/
4812 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
4813 {
4814         struct i40e_pf *pf = vsi->back;
4815         int i, pf_q, ret;
4816
4817         pf_q = vsi->base_queue;
4818         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4819                 /* Check and wait for the Tx queue */
4820                 ret = i40e_pf_txq_wait(pf, pf_q, false);
4821                 if (ret) {
4822                         dev_info(&pf->pdev->dev,
4823                                  "VSI seid %d Tx ring %d disable timeout\n",
4824                                  vsi->seid, pf_q);
4825                         return ret;
4826                 }
4827
4828                 if (!i40e_enabled_xdp_vsi(vsi))
4829                         goto wait_rx;
4830
4831                 /* Check and wait for the XDP Tx queue */
4832                 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
4833                                        false);
4834                 if (ret) {
4835                         dev_info(&pf->pdev->dev,
4836                                  "VSI seid %d XDP Tx ring %d disable timeout\n",
4837                                  vsi->seid, pf_q);
4838                         return ret;
4839                 }
4840 wait_rx:
4841                 /* Check and wait for the Rx queue */
4842                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
4843                 if (ret) {
4844                         dev_info(&pf->pdev->dev,
4845                                  "VSI seid %d Rx ring %d disable timeout\n",
4846                                  vsi->seid, pf_q);
4847                         return ret;
4848                 }
4849         }
4850
4851         return 0;
4852 }
4853
4854 #ifdef CONFIG_I40E_DCB
4855 /**
4856  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
4857  * @pf: the PF
4858  *
4859  * This function waits for the queues to be in disabled state for all the
4860  * VSIs that are managed by this PF.
4861  **/
4862 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
4863 {
4864         int v, ret = 0;
4865
4866         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4867                 if (pf->vsi[v]) {
4868                         ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
4869                         if (ret)
4870                                 break;
4871                 }
4872         }
4873
4874         return ret;
4875 }
4876
4877 #endif
4878
4879 /**
4880  * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue
4881  * @q_idx: TX queue number
4882  * @vsi: Pointer to VSI struct
4883  *
4884  * This function checks specified queue for given VSI. Detects hung condition.
4885  * We proactively detect hung TX queues by checking if interrupts are disabled
4886  * but there are pending descriptors.  If it appears hung, attempt to recover
4887  * by triggering a SW interrupt.
4888  **/
4889 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi)
4890 {
4891         struct i40e_ring *tx_ring = NULL;
4892         struct i40e_pf  *pf;
4893         u32 val, tx_pending;
4894         int i;
4895
4896         pf = vsi->back;
4897
4898         /* now that we have an index, find the tx_ring struct */
4899         for (i = 0; i < vsi->num_queue_pairs; i++) {
4900                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
4901                         if (q_idx == vsi->tx_rings[i]->queue_index) {
4902                                 tx_ring = vsi->tx_rings[i];
4903                                 break;
4904                         }
4905                 }
4906         }
4907
4908         if (!tx_ring)
4909                 return;
4910
4911         /* Read interrupt register */
4912         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4913                 val = rd32(&pf->hw,
4914                            I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
4915                                                tx_ring->vsi->base_vector - 1));
4916         else
4917                 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
4918
4919         tx_pending = i40e_get_tx_pending(tx_ring);
4920
4921         /* Interrupts are disabled and TX pending is non-zero,
4922          * trigger the SW interrupt (don't wait). Worst case
4923          * there will be one extra interrupt which may result
4924          * into not cleaning any queues because queues are cleaned.
4925          */
4926         if (tx_pending && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK)))
4927                 i40e_force_wb(vsi, tx_ring->q_vector);
4928 }
4929
4930 /**
4931  * i40e_detect_recover_hung - Function to detect and recover hung_queues
4932  * @pf:  pointer to PF struct
4933  *
4934  * LAN VSI has netdev and netdev has TX queues. This function is to check
4935  * each of those TX queues if they are hung, trigger recovery by issuing
4936  * SW interrupt.
4937  **/
4938 static void i40e_detect_recover_hung(struct i40e_pf *pf)
4939 {
4940         struct net_device *netdev;
4941         struct i40e_vsi *vsi;
4942         unsigned int i;
4943
4944         /* Only for LAN VSI */
4945         vsi = pf->vsi[pf->lan_vsi];
4946
4947         if (!vsi)
4948                 return;
4949
4950         /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */
4951         if (test_bit(__I40E_VSI_DOWN, vsi->back->state) ||
4952             test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state))
4953                 return;
4954
4955         /* Make sure type is MAIN VSI */
4956         if (vsi->type != I40E_VSI_MAIN)
4957                 return;
4958
4959         netdev = vsi->netdev;
4960         if (!netdev)
4961                 return;
4962
4963         /* Bail out if netif_carrier is not OK */
4964         if (!netif_carrier_ok(netdev))
4965                 return;
4966
4967         /* Go thru' TX queues for netdev */
4968         for (i = 0; i < netdev->num_tx_queues; i++) {
4969                 struct netdev_queue *q;
4970
4971                 q = netdev_get_tx_queue(netdev, i);
4972                 if (q)
4973                         i40e_detect_recover_hung_queue(i, vsi);
4974         }
4975 }
4976
4977 /**
4978  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
4979  * @pf: pointer to PF
4980  *
4981  * Get TC map for ISCSI PF type that will include iSCSI TC
4982  * and LAN TC.
4983  **/
4984 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
4985 {
4986         struct i40e_dcb_app_priority_table app;
4987         struct i40e_hw *hw = &pf->hw;
4988         u8 enabled_tc = 1; /* TC0 is always enabled */
4989         u8 tc, i;
4990         /* Get the iSCSI APP TLV */
4991         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4992
4993         for (i = 0; i < dcbcfg->numapps; i++) {
4994                 app = dcbcfg->app[i];
4995                 if (app.selector == I40E_APP_SEL_TCPIP &&
4996                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
4997                         tc = dcbcfg->etscfg.prioritytable[app.priority];
4998                         enabled_tc |= BIT(tc);
4999                         break;
5000                 }
5001         }
5002
5003         return enabled_tc;
5004 }
5005
5006 /**
5007  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
5008  * @dcbcfg: the corresponding DCBx configuration structure
5009  *
5010  * Return the number of TCs from given DCBx configuration
5011  **/
5012 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5013 {
5014         int i, tc_unused = 0;
5015         u8 num_tc = 0;
5016         u8 ret = 0;
5017
5018         /* Scan the ETS Config Priority Table to find
5019          * traffic class enabled for a given priority
5020          * and create a bitmask of enabled TCs
5021          */
5022         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5023                 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5024
5025         /* Now scan the bitmask to check for
5026          * contiguous TCs starting with TC0
5027          */
5028         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5029                 if (num_tc & BIT(i)) {
5030                         if (!tc_unused) {
5031                                 ret++;
5032                         } else {
5033                                 pr_err("Non-contiguous TC - Disabling DCB\n");
5034                                 return 1;
5035                         }
5036                 } else {
5037                         tc_unused = 1;
5038                 }
5039         }
5040
5041         /* There is always at least TC0 */
5042         if (!ret)
5043                 ret = 1;
5044
5045         return ret;
5046 }
5047
5048 /**
5049  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5050  * @dcbcfg: the corresponding DCBx configuration structure
5051  *
5052  * Query the current DCB configuration and return the number of
5053  * traffic classes enabled from the given DCBX config
5054  **/
5055 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5056 {
5057         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5058         u8 enabled_tc = 1;
5059         u8 i;
5060
5061         for (i = 0; i < num_tc; i++)
5062                 enabled_tc |= BIT(i);
5063
5064         return enabled_tc;
5065 }
5066
5067 /**
5068  * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5069  * @pf: PF being queried
5070  *
5071  * Query the current MQPRIO configuration and return the number of
5072  * traffic classes enabled.
5073  **/
5074 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5075 {
5076         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
5077         u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5078         u8 enabled_tc = 1, i;
5079
5080         for (i = 1; i < num_tc; i++)
5081                 enabled_tc |= BIT(i);
5082         return enabled_tc;
5083 }
5084
5085 /**
5086  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5087  * @pf: PF being queried
5088  *
5089  * Return number of traffic classes enabled for the given PF
5090  **/
5091 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5092 {
5093         struct i40e_hw *hw = &pf->hw;
5094         u8 i, enabled_tc = 1;
5095         u8 num_tc = 0;
5096         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5097
5098         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5099                 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
5100
5101         /* If neither MQPRIO nor DCB is enabled, then always use single TC */
5102         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5103                 return 1;
5104
5105         /* SFP mode will be enabled for all TCs on port */
5106         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5107                 return i40e_dcb_get_num_tc(dcbcfg);
5108
5109         /* MFP mode return count of enabled TCs for this PF */
5110         if (pf->hw.func_caps.iscsi)
5111                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
5112         else
5113                 return 1; /* Only TC0 */
5114
5115         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5116                 if (enabled_tc & BIT(i))
5117                         num_tc++;
5118         }
5119         return num_tc;
5120 }
5121
5122 /**
5123  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
5124  * @pf: PF being queried
5125  *
5126  * Return a bitmap for enabled traffic classes for this PF.
5127  **/
5128 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5129 {
5130         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5131                 return i40e_mqprio_get_enabled_tc(pf);
5132
5133         /* If neither MQPRIO nor DCB is enabled for this PF then just return
5134          * default TC
5135          */
5136         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5137                 return I40E_DEFAULT_TRAFFIC_CLASS;
5138
5139         /* SFP mode we want PF to be enabled for all TCs */
5140         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5141                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5142
5143         /* MFP enabled and iSCSI PF type */
5144         if (pf->hw.func_caps.iscsi)
5145                 return i40e_get_iscsi_tc_map(pf);
5146         else
5147                 return I40E_DEFAULT_TRAFFIC_CLASS;
5148 }
5149
5150 /**
5151  * i40e_vsi_get_bw_info - Query VSI BW Information
5152  * @vsi: the VSI being queried
5153  *
5154  * Returns 0 on success, negative value on failure
5155  **/
5156 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5157 {
5158         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5159         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5160         struct i40e_pf *pf = vsi->back;
5161         struct i40e_hw *hw = &pf->hw;
5162         i40e_status ret;
5163         u32 tc_bw_max;
5164         int i;
5165
5166         /* Get the VSI level BW configuration */
5167         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5168         if (ret) {
5169                 dev_info(&pf->pdev->dev,
5170                          "couldn't get PF vsi bw config, err %s aq_err %s\n",
5171                          i40e_stat_str(&pf->hw, ret),
5172                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5173                 return -EINVAL;
5174         }
5175
5176         /* Get the VSI level BW configuration per TC */
5177         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5178                                                NULL);
5179         if (ret) {
5180                 dev_info(&pf->pdev->dev,
5181                          "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
5182                          i40e_stat_str(&pf->hw, ret),
5183                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5184                 return -EINVAL;
5185         }
5186
5187         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5188                 dev_info(&pf->pdev->dev,
5189                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5190                          bw_config.tc_valid_bits,
5191                          bw_ets_config.tc_valid_bits);
5192                 /* Still continuing */
5193         }
5194
5195         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5196         vsi->bw_max_quanta = bw_config.max_bw;
5197         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5198                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5199         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5200                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5201                 vsi->bw_ets_limit_credits[i] =
5202                                         le16_to_cpu(bw_ets_config.credits[i]);
5203                 /* 3 bits out of 4 for each TC */
5204                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5205         }
5206
5207         return 0;
5208 }
5209
5210 /**
5211  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5212  * @vsi: the VSI being configured
5213  * @enabled_tc: TC bitmap
5214  * @bw_credits: BW shared credits per TC
5215  *
5216  * Returns 0 on success, negative value on failure
5217  **/
5218 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5219                                        u8 *bw_share)
5220 {
5221         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5222         i40e_status ret;
5223         int i;
5224
5225         if (vsi->back->flags & I40E_FLAG_TC_MQPRIO)
5226                 return 0;
5227         if (!vsi->mqprio_qopt.qopt.hw) {
5228                 ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5229                 if (ret)
5230                         dev_info(&vsi->back->pdev->dev,
5231                                  "Failed to reset tx rate for vsi->seid %u\n",
5232                                  vsi->seid);
5233                 return ret;
5234         }
5235         bw_data.tc_valid_bits = enabled_tc;
5236         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5237                 bw_data.tc_bw_credits[i] = bw_share[i];
5238
5239         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
5240                                        NULL);
5241         if (ret) {
5242                 dev_info(&vsi->back->pdev->dev,
5243                          "AQ command Config VSI BW allocation per TC failed = %d\n",
5244                          vsi->back->hw.aq.asq_last_status);
5245                 return -EINVAL;
5246         }
5247
5248         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5249                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5250
5251         return 0;
5252 }
5253
5254 /**
5255  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5256  * @vsi: the VSI being configured
5257  * @enabled_tc: TC map to be enabled
5258  *
5259  **/
5260 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5261 {
5262         struct net_device *netdev = vsi->netdev;
5263         struct i40e_pf *pf = vsi->back;
5264         struct i40e_hw *hw = &pf->hw;
5265         u8 netdev_tc = 0;
5266         int i;
5267         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5268
5269         if (!netdev)
5270                 return;
5271
5272         if (!enabled_tc) {
5273                 netdev_reset_tc(netdev);
5274                 return;
5275         }
5276
5277         /* Set up actual enabled TCs on the VSI */
5278         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5279                 return;
5280
5281         /* set per TC queues for the VSI */
5282         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5283                 /* Only set TC queues for enabled tcs
5284                  *
5285                  * e.g. For a VSI that has TC0 and TC3 enabled the
5286                  * enabled_tc bitmap would be 0x00001001; the driver
5287                  * will set the numtc for netdev as 2 that will be
5288                  * referenced by the netdev layer as TC 0 and 1.
5289                  */
5290                 if (vsi->tc_config.enabled_tc & BIT(i))
5291                         netdev_set_tc_queue(netdev,
5292                                         vsi->tc_config.tc_info[i].netdev_tc,
5293                                         vsi->tc_config.tc_info[i].qcount,
5294                                         vsi->tc_config.tc_info[i].qoffset);
5295         }
5296
5297         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5298                 return;
5299
5300         /* Assign UP2TC map for the VSI */
5301         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5302                 /* Get the actual TC# for the UP */
5303                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5304                 /* Get the mapped netdev TC# for the UP */
5305                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
5306                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5307         }
5308 }
5309
5310 /**
5311  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5312  * @vsi: the VSI being configured
5313  * @ctxt: the ctxt buffer returned from AQ VSI update param command
5314  **/
5315 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5316                                       struct i40e_vsi_context *ctxt)
5317 {
5318         /* copy just the sections touched not the entire info
5319          * since not all sections are valid as returned by
5320          * update vsi params
5321          */
5322         vsi->info.mapping_flags = ctxt->info.mapping_flags;
5323         memcpy(&vsi->info.queue_mapping,
5324                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5325         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5326                sizeof(vsi->info.tc_mapping));
5327 }
5328
5329 /**
5330  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5331  * @vsi: VSI to be configured
5332  * @enabled_tc: TC bitmap
5333  *
5334  * This configures a particular VSI for TCs that are mapped to the
5335  * given TC bitmap. It uses default bandwidth share for TCs across
5336  * VSIs to configure TC for a particular VSI.
5337  *
5338  * NOTE:
5339  * It is expected that the VSI queues have been quisced before calling
5340  * this function.
5341  **/
5342 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5343 {
5344         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5345         struct i40e_vsi_context ctxt;
5346         int ret = 0;
5347         int i;
5348
5349         /* Check if enabled_tc is same as existing or new TCs */
5350         if (vsi->tc_config.enabled_tc == enabled_tc &&
5351             vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5352                 return ret;
5353
5354         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5355         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5356                 if (enabled_tc & BIT(i))
5357                         bw_share[i] = 1;
5358         }
5359
5360         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5361         if (ret) {
5362                 dev_info(&vsi->back->pdev->dev,
5363                          "Failed configuring TC map %d for VSI %d\n",
5364                          enabled_tc, vsi->seid);
5365                 goto out;
5366         }
5367
5368         /* Update Queue Pairs Mapping for currently enabled UPs */
5369         ctxt.seid = vsi->seid;
5370         ctxt.pf_num = vsi->back->hw.pf_id;
5371         ctxt.vf_num = 0;
5372         ctxt.uplink_seid = vsi->uplink_seid;
5373         ctxt.info = vsi->info;
5374         if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) {
5375                 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5376                 if (ret)
5377                         goto out;
5378         } else {
5379                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5380         }
5381
5382         /* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5383          * queues changed.
5384          */
5385         if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5386                 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5387                                       vsi->num_queue_pairs);
5388                 ret = i40e_vsi_config_rss(vsi);
5389                 if (ret) {
5390                         dev_info(&vsi->back->pdev->dev,
5391                                  "Failed to reconfig rss for num_queues\n");
5392                         return ret;
5393                 }
5394                 vsi->reconfig_rss = false;
5395         }
5396         if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5397                 ctxt.info.valid_sections |=
5398                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5399                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5400         }
5401
5402         /* Update the VSI after updating the VSI queue-mapping
5403          * information
5404          */
5405         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
5406         if (ret) {
5407                 dev_info(&vsi->back->pdev->dev,
5408                          "Update vsi tc config failed, err %s aq_err %s\n",
5409                          i40e_stat_str(&vsi->back->hw, ret),
5410                          i40e_aq_str(&vsi->back->hw,
5411                                      vsi->back->hw.aq.asq_last_status));
5412                 goto out;
5413         }
5414         /* update the local VSI info with updated queue map */
5415         i40e_vsi_update_queue_map(vsi, &ctxt);
5416         vsi->info.valid_sections = 0;
5417
5418         /* Update current VSI BW information */
5419         ret = i40e_vsi_get_bw_info(vsi);
5420         if (ret) {
5421                 dev_info(&vsi->back->pdev->dev,
5422                          "Failed updating vsi bw info, err %s aq_err %s\n",
5423                          i40e_stat_str(&vsi->back->hw, ret),
5424                          i40e_aq_str(&vsi->back->hw,
5425                                      vsi->back->hw.aq.asq_last_status));
5426                 goto out;
5427         }
5428
5429         /* Update the netdev TC setup */
5430         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5431 out:
5432         return ret;
5433 }
5434
5435 /**
5436  * i40e_get_link_speed - Returns link speed for the interface
5437  * @vsi: VSI to be configured
5438  *
5439  **/
5440 int i40e_get_link_speed(struct i40e_vsi *vsi)
5441 {
5442         struct i40e_pf *pf = vsi->back;
5443
5444         switch (pf->hw.phy.link_info.link_speed) {
5445         case I40E_LINK_SPEED_40GB:
5446                 return 40000;
5447         case I40E_LINK_SPEED_25GB:
5448                 return 25000;
5449         case I40E_LINK_SPEED_20GB:
5450                 return 20000;
5451         case I40E_LINK_SPEED_10GB:
5452                 return 10000;
5453         case I40E_LINK_SPEED_1GB:
5454                 return 1000;
5455         default:
5456                 return -EINVAL;
5457         }
5458 }
5459
5460 /**
5461  * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5462  * @vsi: VSI to be configured
5463  * @seid: seid of the channel/VSI
5464  * @max_tx_rate: max TX rate to be configured as BW limit
5465  *
5466  * Helper function to set BW limit for a given VSI
5467  **/
5468 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5469 {
5470         struct i40e_pf *pf = vsi->back;
5471         u64 credits = 0;
5472         int speed = 0;
5473         int ret = 0;
5474
5475         speed = i40e_get_link_speed(vsi);
5476         if (max_tx_rate > speed) {
5477                 dev_err(&pf->pdev->dev,
5478                         "Invalid max tx rate %llu specified for VSI seid %d.",
5479                         max_tx_rate, seid);
5480                 return -EINVAL;
5481         }
5482         if (max_tx_rate && max_tx_rate < 50) {
5483                 dev_warn(&pf->pdev->dev,
5484                          "Setting max tx rate to minimum usable value of 50Mbps.\n");
5485                 max_tx_rate = 50;
5486         }
5487
5488         /* Tx rate credits are in values of 50Mbps, 0 is disabled */
5489         credits = max_tx_rate;
5490         do_div(credits, I40E_BW_CREDIT_DIVISOR);
5491         ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
5492                                           I40E_MAX_BW_INACTIVE_ACCUM, NULL);
5493         if (ret)
5494                 dev_err(&pf->pdev->dev,
5495                         "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n",
5496                         max_tx_rate, seid, i40e_stat_str(&pf->hw, ret),
5497                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5498         return ret;
5499 }
5500
5501 /**
5502  * i40e_remove_queue_channels - Remove queue channels for the TCs
5503  * @vsi: VSI to be configured
5504  *
5505  * Remove queue channels for the TCs
5506  **/
5507 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
5508 {
5509         enum i40e_admin_queue_err last_aq_status;
5510         struct i40e_cloud_filter *cfilter;
5511         struct i40e_channel *ch, *ch_tmp;
5512         struct i40e_pf *pf = vsi->back;
5513         struct hlist_node *node;
5514         int ret, i;
5515
5516         /* Reset rss size that was stored when reconfiguring rss for
5517          * channel VSIs with non-power-of-2 queue count.
5518          */
5519         vsi->current_rss_size = 0;
5520
5521         /* perform cleanup for channels if they exist */
5522         if (list_empty(&vsi->ch_list))
5523                 return;
5524
5525         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5526                 struct i40e_vsi *p_vsi;
5527
5528                 list_del(&ch->list);
5529                 p_vsi = ch->parent_vsi;
5530                 if (!p_vsi || !ch->initialized) {
5531                         kfree(ch);
5532                         continue;
5533                 }
5534                 /* Reset queue contexts */
5535                 for (i = 0; i < ch->num_queue_pairs; i++) {
5536                         struct i40e_ring *tx_ring, *rx_ring;
5537                         u16 pf_q;
5538
5539                         pf_q = ch->base_queue + i;
5540                         tx_ring = vsi->tx_rings[pf_q];
5541                         tx_ring->ch = NULL;
5542
5543                         rx_ring = vsi->rx_rings[pf_q];
5544                         rx_ring->ch = NULL;
5545                 }
5546
5547                 /* Reset BW configured for this VSI via mqprio */
5548                 ret = i40e_set_bw_limit(vsi, ch->seid, 0);
5549                 if (ret)
5550                         dev_info(&vsi->back->pdev->dev,
5551                                  "Failed to reset tx rate for ch->seid %u\n",
5552                                  ch->seid);
5553
5554                 /* delete cloud filters associated with this channel */
5555                 hlist_for_each_entry_safe(cfilter, node,
5556                                           &pf->cloud_filter_list, cloud_node) {
5557                         if (cfilter->seid != ch->seid)
5558                                 continue;
5559
5560                         hash_del(&cfilter->cloud_node);
5561                         if (cfilter->dst_port)
5562                                 ret = i40e_add_del_cloud_filter_big_buf(vsi,
5563                                                                         cfilter,
5564                                                                         false);
5565                         else
5566                                 ret = i40e_add_del_cloud_filter(vsi, cfilter,
5567                                                                 false);
5568                         last_aq_status = pf->hw.aq.asq_last_status;
5569                         if (ret)
5570                                 dev_info(&pf->pdev->dev,
5571                                          "Failed to delete cloud filter, err %s aq_err %s\n",
5572                                          i40e_stat_str(&pf->hw, ret),
5573                                          i40e_aq_str(&pf->hw, last_aq_status));
5574                         kfree(cfilter);
5575                 }
5576
5577                 /* delete VSI from FW */
5578                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
5579                                              NULL);
5580                 if (ret)
5581                         dev_err(&vsi->back->pdev->dev,
5582                                 "unable to remove channel (%d) for parent VSI(%d)\n",
5583                                 ch->seid, p_vsi->seid);
5584                 kfree(ch);
5585         }
5586         INIT_LIST_HEAD(&vsi->ch_list);
5587 }
5588
5589 /**
5590  * i40e_is_any_channel - channel exist or not
5591  * @vsi: ptr to VSI to which channels are associated with
5592  *
5593  * Returns true or false if channel(s) exist for associated VSI or not
5594  **/
5595 static bool i40e_is_any_channel(struct i40e_vsi *vsi)
5596 {
5597         struct i40e_channel *ch, *ch_tmp;
5598
5599         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5600                 if (ch->initialized)
5601                         return true;
5602         }
5603
5604         return false;
5605 }
5606
5607 /**
5608  * i40e_get_max_queues_for_channel
5609  * @vsi: ptr to VSI to which channels are associated with
5610  *
5611  * Helper function which returns max value among the queue counts set on the
5612  * channels/TCs created.
5613  **/
5614 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
5615 {
5616         struct i40e_channel *ch, *ch_tmp;
5617         int max = 0;
5618
5619         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5620                 if (!ch->initialized)
5621                         continue;
5622                 if (ch->num_queue_pairs > max)
5623                         max = ch->num_queue_pairs;
5624         }
5625
5626         return max;
5627 }
5628
5629 /**
5630  * i40e_validate_num_queues - validate num_queues w.r.t channel
5631  * @pf: ptr to PF device
5632  * @num_queues: number of queues
5633  * @vsi: the parent VSI
5634  * @reconfig_rss: indicates should the RSS be reconfigured or not
5635  *
5636  * This function validates number of queues in the context of new channel
5637  * which is being established and determines if RSS should be reconfigured
5638  * or not for parent VSI.
5639  **/
5640 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
5641                                     struct i40e_vsi *vsi, bool *reconfig_rss)
5642 {
5643         int max_ch_queues;
5644
5645         if (!reconfig_rss)
5646                 return -EINVAL;
5647
5648         *reconfig_rss = false;
5649         if (vsi->current_rss_size) {
5650                 if (num_queues > vsi->current_rss_size) {
5651                         dev_dbg(&pf->pdev->dev,
5652                                 "Error: num_queues (%d) > vsi's current_size(%d)\n",
5653                                 num_queues, vsi->current_rss_size);
5654                         return -EINVAL;
5655                 } else if ((num_queues < vsi->current_rss_size) &&
5656                            (!is_power_of_2(num_queues))) {
5657                         dev_dbg(&pf->pdev->dev,
5658                                 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
5659                                 num_queues, vsi->current_rss_size);
5660                         return -EINVAL;
5661                 }
5662         }
5663
5664         if (!is_power_of_2(num_queues)) {
5665                 /* Find the max num_queues configured for channel if channel
5666                  * exist.
5667                  * if channel exist, then enforce 'num_queues' to be more than
5668                  * max ever queues configured for channel.
5669                  */
5670                 max_ch_queues = i40e_get_max_queues_for_channel(vsi);
5671                 if (num_queues < max_ch_queues) {
5672                         dev_dbg(&pf->pdev->dev,
5673                                 "Error: num_queues (%d) < max queues configured for channel(%d)\n",
5674                                 num_queues, max_ch_queues);
5675                         return -EINVAL;
5676                 }
5677                 *reconfig_rss = true;
5678         }
5679
5680         return 0;
5681 }
5682
5683 /**
5684  * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
5685  * @vsi: the VSI being setup
5686  * @rss_size: size of RSS, accordingly LUT gets reprogrammed
5687  *
5688  * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
5689  **/
5690 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
5691 {
5692         struct i40e_pf *pf = vsi->back;
5693         u8 seed[I40E_HKEY_ARRAY_SIZE];
5694         struct i40e_hw *hw = &pf->hw;
5695         int local_rss_size;
5696         u8 *lut;
5697         int ret;
5698
5699         if (!vsi->rss_size)
5700                 return -EINVAL;
5701
5702         if (rss_size > vsi->rss_size)
5703                 return -EINVAL;
5704
5705         local_rss_size = min_t(int, vsi->rss_size, rss_size);
5706         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
5707         if (!lut)
5708                 return -ENOMEM;
5709
5710         /* Ignoring user configured lut if there is one */
5711         i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
5712
5713         /* Use user configured hash key if there is one, otherwise
5714          * use default.
5715          */
5716         if (vsi->rss_hkey_user)
5717                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
5718         else
5719                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
5720
5721         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
5722         if (ret) {
5723                 dev_info(&pf->pdev->dev,
5724                          "Cannot set RSS lut, err %s aq_err %s\n",
5725                          i40e_stat_str(hw, ret),
5726                          i40e_aq_str(hw, hw->aq.asq_last_status));
5727                 kfree(lut);
5728                 return ret;
5729         }
5730         kfree(lut);
5731
5732         /* Do the update w.r.t. storing rss_size */
5733         if (!vsi->orig_rss_size)
5734                 vsi->orig_rss_size = vsi->rss_size;
5735         vsi->current_rss_size = local_rss_size;
5736
5737         return ret;
5738 }
5739
5740 /**
5741  * i40e_channel_setup_queue_map - Setup a channel queue map
5742  * @pf: ptr to PF device
5743  * @vsi: the VSI being setup
5744  * @ctxt: VSI context structure
5745  * @ch: ptr to channel structure
5746  *
5747  * Setup queue map for a specific channel
5748  **/
5749 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
5750                                          struct i40e_vsi_context *ctxt,
5751                                          struct i40e_channel *ch)
5752 {
5753         u16 qcount, qmap, sections = 0;
5754         u8 offset = 0;
5755         int pow;
5756
5757         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
5758         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
5759
5760         qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
5761         ch->num_queue_pairs = qcount;
5762
5763         /* find the next higher power-of-2 of num queue pairs */
5764         pow = ilog2(qcount);
5765         if (!is_power_of_2(qcount))
5766                 pow++;
5767
5768         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5769                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
5770
5771         /* Setup queue TC[0].qmap for given VSI context */
5772         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
5773
5774         ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
5775         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
5776         ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
5777         ctxt->info.valid_sections |= cpu_to_le16(sections);
5778 }
5779
5780 /**
5781  * i40e_add_channel - add a channel by adding VSI
5782  * @pf: ptr to PF device
5783  * @uplink_seid: underlying HW switching element (VEB) ID
5784  * @ch: ptr to channel structure
5785  *
5786  * Add a channel (VSI) using add_vsi and queue_map
5787  **/
5788 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
5789                             struct i40e_channel *ch)
5790 {
5791         struct i40e_hw *hw = &pf->hw;
5792         struct i40e_vsi_context ctxt;
5793         u8 enabled_tc = 0x1; /* TC0 enabled */
5794         int ret;
5795
5796         if (ch->type != I40E_VSI_VMDQ2) {
5797                 dev_info(&pf->pdev->dev,
5798                          "add new vsi failed, ch->type %d\n", ch->type);
5799                 return -EINVAL;
5800         }
5801
5802         memset(&ctxt, 0, sizeof(ctxt));
5803         ctxt.pf_num = hw->pf_id;
5804         ctxt.vf_num = 0;
5805         ctxt.uplink_seid = uplink_seid;
5806         ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5807         if (ch->type == I40E_VSI_VMDQ2)
5808                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
5809
5810         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) {
5811                 ctxt.info.valid_sections |=
5812                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5813                 ctxt.info.switch_id =
5814                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
5815         }
5816
5817         /* Set queue map for a given VSI context */
5818         i40e_channel_setup_queue_map(pf, &ctxt, ch);
5819
5820         /* Now time to create VSI */
5821         ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
5822         if (ret) {
5823                 dev_info(&pf->pdev->dev,
5824                          "add new vsi failed, err %s aq_err %s\n",
5825                          i40e_stat_str(&pf->hw, ret),
5826                          i40e_aq_str(&pf->hw,
5827                                      pf->hw.aq.asq_last_status));
5828                 return -ENOENT;
5829         }
5830
5831         /* Success, update channel */
5832         ch->enabled_tc = enabled_tc;
5833         ch->seid = ctxt.seid;
5834         ch->vsi_number = ctxt.vsi_number;
5835         ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx);
5836
5837         /* copy just the sections touched not the entire info
5838          * since not all sections are valid as returned by
5839          * update vsi params
5840          */
5841         ch->info.mapping_flags = ctxt.info.mapping_flags;
5842         memcpy(&ch->info.queue_mapping,
5843                &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
5844         memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
5845                sizeof(ctxt.info.tc_mapping));
5846
5847         return 0;
5848 }
5849
5850 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
5851                                   u8 *bw_share)
5852 {
5853         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5854         i40e_status ret;
5855         int i;
5856
5857         bw_data.tc_valid_bits = ch->enabled_tc;
5858         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5859                 bw_data.tc_bw_credits[i] = bw_share[i];
5860
5861         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
5862                                        &bw_data, NULL);
5863         if (ret) {
5864                 dev_info(&vsi->back->pdev->dev,
5865                          "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
5866                          vsi->back->hw.aq.asq_last_status, ch->seid);
5867                 return -EINVAL;
5868         }
5869
5870         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5871                 ch->info.qs_handle[i] = bw_data.qs_handles[i];
5872
5873         return 0;
5874 }
5875
5876 /**
5877  * i40e_channel_config_tx_ring - config TX ring associated with new channel
5878  * @pf: ptr to PF device
5879  * @vsi: the VSI being setup
5880  * @ch: ptr to channel structure
5881  *
5882  * Configure TX rings associated with channel (VSI) since queues are being
5883  * from parent VSI.
5884  **/
5885 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
5886                                        struct i40e_vsi *vsi,
5887                                        struct i40e_channel *ch)
5888 {
5889         i40e_status ret;
5890         int i;
5891         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5892
5893         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5894         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5895                 if (ch->enabled_tc & BIT(i))
5896                         bw_share[i] = 1;
5897         }
5898
5899         /* configure BW for new VSI */
5900         ret = i40e_channel_config_bw(vsi, ch, bw_share);
5901         if (ret) {
5902                 dev_info(&vsi->back->pdev->dev,
5903                          "Failed configuring TC map %d for channel (seid %u)\n",
5904                          ch->enabled_tc, ch->seid);
5905                 return ret;
5906         }
5907
5908         for (i = 0; i < ch->num_queue_pairs; i++) {
5909                 struct i40e_ring *tx_ring, *rx_ring;
5910                 u16 pf_q;
5911
5912                 pf_q = ch->base_queue + i;
5913
5914                 /* Get to TX ring ptr of main VSI, for re-setup TX queue
5915                  * context
5916                  */
5917                 tx_ring = vsi->tx_rings[pf_q];
5918                 tx_ring->ch = ch;
5919
5920                 /* Get the RX ring ptr */
5921                 rx_ring = vsi->rx_rings[pf_q];
5922                 rx_ring->ch = ch;
5923         }
5924
5925         return 0;
5926 }
5927
5928 /**
5929  * i40e_setup_hw_channel - setup new channel
5930  * @pf: ptr to PF device
5931  * @vsi: the VSI being setup
5932  * @ch: ptr to channel structure
5933  * @uplink_seid: underlying HW switching element (VEB) ID
5934  * @type: type of channel to be created (VMDq2/VF)
5935  *
5936  * Setup new channel (VSI) based on specified type (VMDq2/VF)
5937  * and configures TX rings accordingly
5938  **/
5939 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
5940                                         struct i40e_vsi *vsi,
5941                                         struct i40e_channel *ch,
5942                                         u16 uplink_seid, u8 type)
5943 {
5944         int ret;
5945
5946         ch->initialized = false;
5947         ch->base_queue = vsi->next_base_queue;
5948         ch->type = type;
5949
5950         /* Proceed with creation of channel (VMDq2) VSI */
5951         ret = i40e_add_channel(pf, uplink_seid, ch);
5952         if (ret) {
5953                 dev_info(&pf->pdev->dev,
5954                          "failed to add_channel using uplink_seid %u\n",
5955                          uplink_seid);
5956                 return ret;
5957         }
5958
5959         /* Mark the successful creation of channel */
5960         ch->initialized = true;
5961
5962         /* Reconfigure TX queues using QTX_CTL register */
5963         ret = i40e_channel_config_tx_ring(pf, vsi, ch);
5964         if (ret) {
5965                 dev_info(&pf->pdev->dev,
5966                          "failed to configure TX rings for channel %u\n",
5967                          ch->seid);
5968                 return ret;
5969         }
5970
5971         /* update 'next_base_queue' */
5972         vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
5973         dev_dbg(&pf->pdev->dev,
5974                 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
5975                 ch->seid, ch->vsi_number, ch->stat_counter_idx,
5976                 ch->num_queue_pairs,
5977                 vsi->next_base_queue);
5978         return ret;
5979 }
5980
5981 /**
5982  * i40e_setup_channel - setup new channel using uplink element
5983  * @pf: ptr to PF device
5984  * @type: type of channel to be created (VMDq2/VF)
5985  * @uplink_seid: underlying HW switching element (VEB) ID
5986  * @ch: ptr to channel structure
5987  *
5988  * Setup new channel (VSI) based on specified type (VMDq2/VF)
5989  * and uplink switching element (uplink_seid)
5990  **/
5991 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
5992                                struct i40e_channel *ch)
5993 {
5994         u8 vsi_type;
5995         u16 seid;
5996         int ret;
5997
5998         if (vsi->type == I40E_VSI_MAIN) {
5999                 vsi_type = I40E_VSI_VMDQ2;
6000         } else {
6001                 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6002                         vsi->type);
6003                 return false;
6004         }
6005
6006         /* underlying switching element */
6007         seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6008
6009         /* create channel (VSI), configure TX rings */
6010         ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6011         if (ret) {
6012                 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6013                 return false;
6014         }
6015
6016         return ch->initialized ? true : false;
6017 }
6018
6019 /**
6020  * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6021  * @vsi: ptr to VSI which has PF backing
6022  *
6023  * Sets up switch mode correctly if it needs to be changed and perform
6024  * what are allowed modes.
6025  **/
6026 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6027 {
6028         u8 mode;
6029         struct i40e_pf *pf = vsi->back;
6030         struct i40e_hw *hw = &pf->hw;
6031         int ret;
6032
6033         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6034         if (ret)
6035                 return -EINVAL;
6036
6037         if (hw->dev_caps.switch_mode) {
6038                 /* if switch mode is set, support mode2 (non-tunneled for
6039                  * cloud filter) for now
6040                  */
6041                 u32 switch_mode = hw->dev_caps.switch_mode &
6042                                   I40E_SWITCH_MODE_MASK;
6043                 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6044                         if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6045                                 return 0;
6046                         dev_err(&pf->pdev->dev,
6047                                 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6048                                 hw->dev_caps.switch_mode);
6049                         return -EINVAL;
6050                 }
6051         }
6052
6053         /* Set Bit 7 to be valid */
6054         mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6055
6056         /* Set L4type for TCP support */
6057         mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6058
6059         /* Set cloud filter mode */
6060         mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6061
6062         /* Prep mode field for set_switch_config */
6063         ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6064                                         pf->last_sw_conf_valid_flags,
6065                                         mode, NULL);
6066         if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6067                 dev_err(&pf->pdev->dev,
6068                         "couldn't set switch config bits, err %s aq_err %s\n",
6069                         i40e_stat_str(hw, ret),
6070                         i40e_aq_str(hw,
6071                                     hw->aq.asq_last_status));
6072
6073         return ret;
6074 }
6075
6076 /**
6077  * i40e_create_queue_channel - function to create channel
6078  * @vsi: VSI to be configured
6079  * @ch: ptr to channel (it contains channel specific params)
6080  *
6081  * This function creates channel (VSI) using num_queues specified by user,
6082  * reconfigs RSS if needed.
6083  **/
6084 int i40e_create_queue_channel(struct i40e_vsi *vsi,
6085                               struct i40e_channel *ch)
6086 {
6087         struct i40e_pf *pf = vsi->back;
6088         bool reconfig_rss;
6089         int err;
6090
6091         if (!ch)
6092                 return -EINVAL;
6093
6094         if (!ch->num_queue_pairs) {
6095                 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6096                         ch->num_queue_pairs);
6097                 return -EINVAL;
6098         }
6099
6100         /* validate user requested num_queues for channel */
6101         err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6102                                        &reconfig_rss);
6103         if (err) {
6104                 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6105                          ch->num_queue_pairs);
6106                 return -EINVAL;
6107         }
6108
6109         /* By default we are in VEPA mode, if this is the first VF/VMDq
6110          * VSI to be added switch to VEB mode.
6111          */
6112         if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) ||
6113             (!i40e_is_any_channel(vsi))) {
6114                 if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) {
6115                         dev_dbg(&pf->pdev->dev,
6116                                 "Failed to create channel. Override queues (%u) not power of 2\n",
6117                                 vsi->tc_config.tc_info[0].qcount);
6118                         return -EINVAL;
6119                 }
6120
6121                 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
6122                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
6123
6124                         if (vsi->type == I40E_VSI_MAIN) {
6125                                 if (pf->flags & I40E_FLAG_TC_MQPRIO)
6126                                         i40e_do_reset(pf, I40E_PF_RESET_FLAG,
6127                                                       true);
6128                                 else
6129                                         i40e_do_reset_safe(pf,
6130                                                            I40E_PF_RESET_FLAG);
6131                         }
6132                 }
6133                 /* now onwards for main VSI, number of queues will be value
6134                  * of TC0's queue count
6135                  */
6136         }
6137
6138         /* By this time, vsi->cnt_q_avail shall be set to non-zero and
6139          * it should be more than num_queues
6140          */
6141         if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6142                 dev_dbg(&pf->pdev->dev,
6143                         "Error: cnt_q_avail (%u) less than num_queues %d\n",
6144                         vsi->cnt_q_avail, ch->num_queue_pairs);
6145                 return -EINVAL;
6146         }
6147
6148         /* reconfig_rss only if vsi type is MAIN_VSI */
6149         if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6150                 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6151                 if (err) {
6152                         dev_info(&pf->pdev->dev,
6153                                  "Error: unable to reconfig rss for num_queues (%u)\n",
6154                                  ch->num_queue_pairs);
6155                         return -EINVAL;
6156                 }
6157         }
6158
6159         if (!i40e_setup_channel(pf, vsi, ch)) {
6160                 dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6161                 return -EINVAL;
6162         }
6163
6164         dev_info(&pf->pdev->dev,
6165                  "Setup channel (id:%u) utilizing num_queues %d\n",
6166                  ch->seid, ch->num_queue_pairs);
6167
6168         /* configure VSI for BW limit */
6169         if (ch->max_tx_rate) {
6170                 u64 credits = ch->max_tx_rate;
6171
6172                 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6173                         return -EINVAL;
6174
6175                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6176                 dev_dbg(&pf->pdev->dev,
6177                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6178                         ch->max_tx_rate,
6179                         credits,
6180                         ch->seid);
6181         }
6182
6183         /* in case of VF, this will be main SRIOV VSI */
6184         ch->parent_vsi = vsi;
6185
6186         /* and update main_vsi's count for queue_available to use */
6187         vsi->cnt_q_avail -= ch->num_queue_pairs;
6188
6189         return 0;
6190 }
6191
6192 /**
6193  * i40e_configure_queue_channels - Add queue channel for the given TCs
6194  * @vsi: VSI to be configured
6195  *
6196  * Configures queue channel mapping to the given TCs
6197  **/
6198 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6199 {
6200         struct i40e_channel *ch;
6201         u64 max_rate = 0;
6202         int ret = 0, i;
6203
6204         /* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6205         vsi->tc_seid_map[0] = vsi->seid;
6206         for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6207                 if (vsi->tc_config.enabled_tc & BIT(i)) {
6208                         ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6209                         if (!ch) {
6210                                 ret = -ENOMEM;
6211                                 goto err_free;
6212                         }
6213
6214                         INIT_LIST_HEAD(&ch->list);
6215                         ch->num_queue_pairs =
6216                                 vsi->tc_config.tc_info[i].qcount;
6217                         ch->base_queue =
6218                                 vsi->tc_config.tc_info[i].qoffset;
6219
6220                         /* Bandwidth limit through tc interface is in bytes/s,
6221                          * change to Mbit/s
6222                          */
6223                         max_rate = vsi->mqprio_qopt.max_rate[i];
6224                         do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6225                         ch->max_tx_rate = max_rate;
6226
6227                         list_add_tail(&ch->list, &vsi->ch_list);
6228
6229                         ret = i40e_create_queue_channel(vsi, ch);
6230                         if (ret) {
6231                                 dev_err(&vsi->back->pdev->dev,
6232                                         "Failed creating queue channel with TC%d: queues %d\n",
6233                                         i, ch->num_queue_pairs);
6234                                 goto err_free;
6235                         }
6236                         vsi->tc_seid_map[i] = ch->seid;
6237                 }
6238         }
6239         return ret;
6240
6241 err_free:
6242         i40e_remove_queue_channels(vsi);
6243         return ret;
6244 }
6245
6246 /**
6247  * i40e_veb_config_tc - Configure TCs for given VEB
6248  * @veb: given VEB
6249  * @enabled_tc: TC bitmap
6250  *
6251  * Configures given TC bitmap for VEB (switching) element
6252  **/
6253 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6254 {
6255         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6256         struct i40e_pf *pf = veb->pf;
6257         int ret = 0;
6258         int i;
6259
6260         /* No TCs or already enabled TCs just return */
6261         if (!enabled_tc || veb->enabled_tc == enabled_tc)
6262                 return ret;
6263
6264         bw_data.tc_valid_bits = enabled_tc;
6265         /* bw_data.absolute_credits is not set (relative) */
6266
6267         /* Enable ETS TCs with equal BW Share for now */
6268         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6269                 if (enabled_tc & BIT(i))
6270                         bw_data.tc_bw_share_credits[i] = 1;
6271         }
6272
6273         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6274                                                    &bw_data, NULL);
6275         if (ret) {
6276                 dev_info(&pf->pdev->dev,
6277                          "VEB bw config failed, err %s aq_err %s\n",
6278                          i40e_stat_str(&pf->hw, ret),
6279                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6280                 goto out;
6281         }
6282
6283         /* Update the BW information */
6284         ret = i40e_veb_get_bw_info(veb);
6285         if (ret) {
6286                 dev_info(&pf->pdev->dev,
6287                          "Failed getting veb bw config, err %s aq_err %s\n",
6288                          i40e_stat_str(&pf->hw, ret),
6289                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6290         }
6291
6292 out:
6293         return ret;
6294 }
6295
6296 #ifdef CONFIG_I40E_DCB
6297 /**
6298  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6299  * @pf: PF struct
6300  *
6301  * Reconfigure VEB/VSIs on a given PF; it is assumed that
6302  * the caller would've quiesce all the VSIs before calling
6303  * this function
6304  **/
6305 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6306 {
6307         u8 tc_map = 0;
6308         int ret;
6309         u8 v;
6310
6311         /* Enable the TCs available on PF to all VEBs */
6312         tc_map = i40e_pf_get_tc_map(pf);
6313         for (v = 0; v < I40E_MAX_VEB; v++) {
6314                 if (!pf->veb[v])
6315                         continue;
6316                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
6317                 if (ret) {
6318                         dev_info(&pf->pdev->dev,
6319                                  "Failed configuring TC for VEB seid=%d\n",
6320                                  pf->veb[v]->seid);
6321                         /* Will try to configure as many components */
6322                 }
6323         }
6324
6325         /* Update each VSI */
6326         for (v = 0; v < pf->num_alloc_vsi; v++) {
6327                 if (!pf->vsi[v])
6328                         continue;
6329
6330                 /* - Enable all TCs for the LAN VSI
6331                  * - For all others keep them at TC0 for now
6332                  */
6333                 if (v == pf->lan_vsi)
6334                         tc_map = i40e_pf_get_tc_map(pf);
6335                 else
6336                         tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6337
6338                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
6339                 if (ret) {
6340                         dev_info(&pf->pdev->dev,
6341                                  "Failed configuring TC for VSI seid=%d\n",
6342                                  pf->vsi[v]->seid);
6343                         /* Will try to configure as many components */
6344                 } else {
6345                         /* Re-configure VSI vectors based on updated TC map */
6346                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
6347                         if (pf->vsi[v]->netdev)
6348                                 i40e_dcbnl_set_all(pf->vsi[v]);
6349                 }
6350         }
6351 }
6352
6353 /**
6354  * i40e_resume_port_tx - Resume port Tx
6355  * @pf: PF struct
6356  *
6357  * Resume a port's Tx and issue a PF reset in case of failure to
6358  * resume.
6359  **/
6360 static int i40e_resume_port_tx(struct i40e_pf *pf)
6361 {
6362         struct i40e_hw *hw = &pf->hw;
6363         int ret;
6364
6365         ret = i40e_aq_resume_port_tx(hw, NULL);
6366         if (ret) {
6367                 dev_info(&pf->pdev->dev,
6368                          "Resume Port Tx failed, err %s aq_err %s\n",
6369                           i40e_stat_str(&pf->hw, ret),
6370                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6371                 /* Schedule PF reset to recover */
6372                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6373                 i40e_service_event_schedule(pf);
6374         }
6375
6376         return ret;
6377 }
6378
6379 /**
6380  * i40e_init_pf_dcb - Initialize DCB configuration
6381  * @pf: PF being configured
6382  *
6383  * Query the current DCB configuration and cache it
6384  * in the hardware structure
6385  **/
6386 static int i40e_init_pf_dcb(struct i40e_pf *pf)
6387 {
6388         struct i40e_hw *hw = &pf->hw;
6389         int err = 0;
6390
6391         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */
6392         if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT)
6393                 goto out;
6394
6395         /* Get the initial DCB configuration */
6396         err = i40e_init_dcb(hw);
6397         if (!err) {
6398                 /* Device/Function is not DCBX capable */
6399                 if ((!hw->func_caps.dcb) ||
6400                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
6401                         dev_info(&pf->pdev->dev,
6402                                  "DCBX offload is not supported or is disabled for this PF.\n");
6403                 } else {
6404                         /* When status is not DISABLED then DCBX in FW */
6405                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
6406                                        DCB_CAP_DCBX_VER_IEEE;
6407
6408                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
6409                         /* Enable DCB tagging only when more than one TC
6410                          * or explicitly disable if only one TC
6411                          */
6412                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
6413                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
6414                         else
6415                                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6416                         dev_dbg(&pf->pdev->dev,
6417                                 "DCBX offload is supported for this PF.\n");
6418                 }
6419         } else {
6420                 dev_info(&pf->pdev->dev,
6421                          "Query for DCB configuration failed, err %s aq_err %s\n",
6422                          i40e_stat_str(&pf->hw, err),
6423                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6424         }
6425
6426 out:
6427         return err;
6428 }
6429 #endif /* CONFIG_I40E_DCB */
6430 #define SPEED_SIZE 14
6431 #define FC_SIZE 8
6432 /**
6433  * i40e_print_link_message - print link up or down
6434  * @vsi: the VSI for which link needs a message
6435  */
6436 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
6437 {
6438         enum i40e_aq_link_speed new_speed;
6439         struct i40e_pf *pf = vsi->back;
6440         char *speed = "Unknown";
6441         char *fc = "Unknown";
6442         char *fec = "";
6443         char *req_fec = "";
6444         char *an = "";
6445
6446         new_speed = pf->hw.phy.link_info.link_speed;
6447
6448         if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
6449                 return;
6450         vsi->current_isup = isup;
6451         vsi->current_speed = new_speed;
6452         if (!isup) {
6453                 netdev_info(vsi->netdev, "NIC Link is Down\n");
6454                 return;
6455         }
6456
6457         /* Warn user if link speed on NPAR enabled partition is not at
6458          * least 10GB
6459          */
6460         if (pf->hw.func_caps.npar_enable &&
6461             (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
6462              pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
6463                 netdev_warn(vsi->netdev,
6464                             "The partition detected link speed that is less than 10Gbps\n");
6465
6466         switch (pf->hw.phy.link_info.link_speed) {
6467         case I40E_LINK_SPEED_40GB:
6468                 speed = "40 G";
6469                 break;
6470         case I40E_LINK_SPEED_20GB:
6471                 speed = "20 G";
6472                 break;
6473         case I40E_LINK_SPEED_25GB:
6474                 speed = "25 G";
6475                 break;
6476         case I40E_LINK_SPEED_10GB:
6477                 speed = "10 G";
6478                 break;
6479         case I40E_LINK_SPEED_1GB:
6480                 speed = "1000 M";
6481                 break;
6482         case I40E_LINK_SPEED_100MB:
6483                 speed = "100 M";
6484                 break;
6485         default:
6486                 break;
6487         }
6488
6489         switch (pf->hw.fc.current_mode) {
6490         case I40E_FC_FULL:
6491                 fc = "RX/TX";
6492                 break;
6493         case I40E_FC_TX_PAUSE:
6494                 fc = "TX";
6495                 break;
6496         case I40E_FC_RX_PAUSE:
6497                 fc = "RX";
6498                 break;
6499         default:
6500                 fc = "None";
6501                 break;
6502         }
6503
6504         if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
6505                 req_fec = ", Requested FEC: None";
6506                 fec = ", FEC: None";
6507                 an = ", Autoneg: False";
6508
6509                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
6510                         an = ", Autoneg: True";
6511
6512                 if (pf->hw.phy.link_info.fec_info &
6513                     I40E_AQ_CONFIG_FEC_KR_ENA)
6514                         fec = ", FEC: CL74 FC-FEC/BASE-R";
6515                 else if (pf->hw.phy.link_info.fec_info &
6516                          I40E_AQ_CONFIG_FEC_RS_ENA)
6517                         fec = ", FEC: CL108 RS-FEC";
6518
6519                 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
6520                  * both RS and FC are requested
6521                  */
6522                 if (vsi->back->hw.phy.link_info.req_fec_info &
6523                     (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
6524                         if (vsi->back->hw.phy.link_info.req_fec_info &
6525                             I40E_AQ_REQUEST_FEC_RS)
6526                                 req_fec = ", Requested FEC: CL108 RS-FEC";
6527                         else
6528                                 req_fec = ", Requested FEC: CL74 FC-FEC/BASE-R";
6529                 }
6530         }
6531
6532         netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s%s, Flow Control: %s\n",
6533                     speed, req_fec, fec, an, fc);
6534 }
6535
6536 /**
6537  * i40e_up_complete - Finish the last steps of bringing up a connection
6538  * @vsi: the VSI being configured
6539  **/
6540 static int i40e_up_complete(struct i40e_vsi *vsi)
6541 {
6542         struct i40e_pf *pf = vsi->back;
6543         int err;
6544
6545         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
6546                 i40e_vsi_configure_msix(vsi);
6547         else
6548                 i40e_configure_msi_and_legacy(vsi);
6549
6550         /* start rings */
6551         err = i40e_vsi_start_rings(vsi);
6552         if (err)
6553                 return err;
6554
6555         clear_bit(__I40E_VSI_DOWN, vsi->state);
6556         i40e_napi_enable_all(vsi);
6557         i40e_vsi_enable_irq(vsi);
6558
6559         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
6560             (vsi->netdev)) {
6561                 i40e_print_link_message(vsi, true);
6562                 netif_tx_start_all_queues(vsi->netdev);
6563                 netif_carrier_on(vsi->netdev);
6564         }
6565
6566         /* replay FDIR SB filters */
6567         if (vsi->type == I40E_VSI_FDIR) {
6568                 /* reset fd counters */
6569                 pf->fd_add_err = 0;
6570                 pf->fd_atr_cnt = 0;
6571                 i40e_fdir_filter_restore(vsi);
6572         }
6573
6574         /* On the next run of the service_task, notify any clients of the new
6575          * opened netdev
6576          */
6577         pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
6578         i40e_service_event_schedule(pf);
6579
6580         return 0;
6581 }
6582
6583 /**
6584  * i40e_vsi_reinit_locked - Reset the VSI
6585  * @vsi: the VSI being configured
6586  *
6587  * Rebuild the ring structs after some configuration
6588  * has changed, e.g. MTU size.
6589  **/
6590 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
6591 {
6592         struct i40e_pf *pf = vsi->back;
6593
6594         WARN_ON(in_interrupt());
6595         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
6596                 usleep_range(1000, 2000);
6597         i40e_down(vsi);
6598
6599         i40e_up(vsi);
6600         clear_bit(__I40E_CONFIG_BUSY, pf->state);
6601 }
6602
6603 /**
6604  * i40e_up - Bring the connection back up after being down
6605  * @vsi: the VSI being configured
6606  **/
6607 int i40e_up(struct i40e_vsi *vsi)
6608 {
6609         int err;
6610
6611         err = i40e_vsi_configure(vsi);
6612         if (!err)
6613                 err = i40e_up_complete(vsi);
6614
6615         return err;
6616 }
6617
6618 /**
6619  * i40e_down - Shutdown the connection processing
6620  * @vsi: the VSI being stopped
6621  **/
6622 void i40e_down(struct i40e_vsi *vsi)
6623 {
6624         int i;
6625
6626         /* It is assumed that the caller of this function
6627          * sets the vsi->state __I40E_VSI_DOWN bit.
6628          */
6629         if (vsi->netdev) {
6630                 netif_carrier_off(vsi->netdev);
6631                 netif_tx_disable(vsi->netdev);
6632         }
6633         i40e_vsi_disable_irq(vsi);
6634         i40e_vsi_stop_rings(vsi);
6635         i40e_napi_disable_all(vsi);
6636
6637         for (i = 0; i < vsi->num_queue_pairs; i++) {
6638                 i40e_clean_tx_ring(vsi->tx_rings[i]);
6639                 if (i40e_enabled_xdp_vsi(vsi))
6640                         i40e_clean_tx_ring(vsi->xdp_rings[i]);
6641                 i40e_clean_rx_ring(vsi->rx_rings[i]);
6642         }
6643
6644 }
6645
6646 /**
6647  * i40e_validate_mqprio_qopt- validate queue mapping info
6648  * @vsi: the VSI being configured
6649  * @mqprio_qopt: queue parametrs
6650  **/
6651 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
6652                                      struct tc_mqprio_qopt_offload *mqprio_qopt)
6653 {
6654         u64 sum_max_rate = 0;
6655         u64 max_rate = 0;
6656         int i;
6657
6658         if (mqprio_qopt->qopt.offset[0] != 0 ||
6659             mqprio_qopt->qopt.num_tc < 1 ||
6660             mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
6661                 return -EINVAL;
6662         for (i = 0; ; i++) {
6663                 if (!mqprio_qopt->qopt.count[i])
6664                         return -EINVAL;
6665                 if (mqprio_qopt->min_rate[i]) {
6666                         dev_err(&vsi->back->pdev->dev,
6667                                 "Invalid min tx rate (greater than 0) specified\n");
6668                         return -EINVAL;
6669                 }
6670                 max_rate = mqprio_qopt->max_rate[i];
6671                 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6672                 sum_max_rate += max_rate;
6673
6674                 if (i >= mqprio_qopt->qopt.num_tc - 1)
6675                         break;
6676                 if (mqprio_qopt->qopt.offset[i + 1] !=
6677                     (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
6678                         return -EINVAL;
6679         }
6680         if (vsi->num_queue_pairs <
6681             (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
6682                 return -EINVAL;
6683         }
6684         if (sum_max_rate > i40e_get_link_speed(vsi)) {
6685                 dev_err(&vsi->back->pdev->dev,
6686                         "Invalid max tx rate specified\n");
6687                 return -EINVAL;
6688         }
6689         return 0;
6690 }
6691
6692 /**
6693  * i40e_vsi_set_default_tc_config - set default values for tc configuration
6694  * @vsi: the VSI being configured
6695  **/
6696 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
6697 {
6698         u16 qcount;
6699         int i;
6700
6701         /* Only TC0 is enabled */
6702         vsi->tc_config.numtc = 1;
6703         vsi->tc_config.enabled_tc = 1;
6704         qcount = min_t(int, vsi->alloc_queue_pairs,
6705                        i40e_pf_get_max_q_per_tc(vsi->back));
6706         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6707                 /* For the TC that is not enabled set the offset to to default
6708                  * queue and allocate one queue for the given TC.
6709                  */
6710                 vsi->tc_config.tc_info[i].qoffset = 0;
6711                 if (i == 0)
6712                         vsi->tc_config.tc_info[i].qcount = qcount;
6713                 else
6714                         vsi->tc_config.tc_info[i].qcount = 1;
6715                 vsi->tc_config.tc_info[i].netdev_tc = 0;
6716         }
6717 }
6718
6719 /**
6720  * i40e_setup_tc - configure multiple traffic classes
6721  * @netdev: net device to configure
6722  * @type_data: tc offload data
6723  **/
6724 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
6725 {
6726         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
6727         struct i40e_netdev_priv *np = netdev_priv(netdev);
6728         struct i40e_vsi *vsi = np->vsi;
6729         struct i40e_pf *pf = vsi->back;
6730         u8 enabled_tc = 0, num_tc, hw;
6731         bool need_reset = false;
6732         int ret = -EINVAL;
6733         u16 mode;
6734         int i;
6735
6736         num_tc = mqprio_qopt->qopt.num_tc;
6737         hw = mqprio_qopt->qopt.hw;
6738         mode = mqprio_qopt->mode;
6739         if (!hw) {
6740                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
6741                 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
6742                 goto config_tc;
6743         }
6744
6745         /* Check if MFP enabled */
6746         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
6747                 netdev_info(netdev,
6748                             "Configuring TC not supported in MFP mode\n");
6749                 return ret;
6750         }
6751         switch (mode) {
6752         case TC_MQPRIO_MODE_DCB:
6753                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
6754
6755                 /* Check if DCB enabled to continue */
6756                 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
6757                         netdev_info(netdev,
6758                                     "DCB is not enabled for adapter\n");
6759                         return ret;
6760                 }
6761
6762                 /* Check whether tc count is within enabled limit */
6763                 if (num_tc > i40e_pf_get_num_tc(pf)) {
6764                         netdev_info(netdev,
6765                                     "TC count greater than enabled on link for adapter\n");
6766                         return ret;
6767                 }
6768                 break;
6769         case TC_MQPRIO_MODE_CHANNEL:
6770                 if (pf->flags & I40E_FLAG_DCB_ENABLED) {
6771                         netdev_info(netdev,
6772                                     "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
6773                         return ret;
6774                 }
6775                 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
6776                         return ret;
6777                 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
6778                 if (ret)
6779                         return ret;
6780                 memcpy(&vsi->mqprio_qopt, mqprio_qopt,
6781                        sizeof(*mqprio_qopt));
6782                 pf->flags |= I40E_FLAG_TC_MQPRIO;
6783                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6784                 break;
6785         default:
6786                 return -EINVAL;
6787         }
6788
6789 config_tc:
6790         /* Generate TC map for number of tc requested */
6791         for (i = 0; i < num_tc; i++)
6792                 enabled_tc |= BIT(i);
6793
6794         /* Requesting same TC configuration as already enabled */
6795         if (enabled_tc == vsi->tc_config.enabled_tc &&
6796             mode != TC_MQPRIO_MODE_CHANNEL)
6797                 return 0;
6798
6799         /* Quiesce VSI queues */
6800         i40e_quiesce_vsi(vsi);
6801
6802         if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO))
6803                 i40e_remove_queue_channels(vsi);
6804
6805         /* Configure VSI for enabled TCs */
6806         ret = i40e_vsi_config_tc(vsi, enabled_tc);
6807         if (ret) {
6808                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
6809                             vsi->seid);
6810                 need_reset = true;
6811                 goto exit;
6812         }
6813
6814         if (pf->flags & I40E_FLAG_TC_MQPRIO) {
6815                 if (vsi->mqprio_qopt.max_rate[0]) {
6816                         u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
6817
6818                         do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
6819                         ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
6820                         if (!ret) {
6821                                 u64 credits = max_tx_rate;
6822
6823                                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6824                                 dev_dbg(&vsi->back->pdev->dev,
6825                                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6826                                         max_tx_rate,
6827                                         credits,
6828                                         vsi->seid);
6829                         } else {
6830                                 need_reset = true;
6831                                 goto exit;
6832                         }
6833                 }
6834                 ret = i40e_configure_queue_channels(vsi);
6835                 if (ret) {
6836                         netdev_info(netdev,
6837                                     "Failed configuring queue channels\n");
6838                         need_reset = true;
6839                         goto exit;
6840                 }
6841         }
6842
6843 exit:
6844         /* Reset the configuration data to defaults, only TC0 is enabled */
6845         if (need_reset) {
6846                 i40e_vsi_set_default_tc_config(vsi);
6847                 need_reset = false;
6848         }
6849
6850         /* Unquiesce VSI */
6851         i40e_unquiesce_vsi(vsi);
6852         return ret;
6853 }
6854
6855 /**
6856  * i40e_set_cld_element - sets cloud filter element data
6857  * @filter: cloud filter rule
6858  * @cld: ptr to cloud filter element data
6859  *
6860  * This is helper function to copy data into cloud filter element
6861  **/
6862 static inline void
6863 i40e_set_cld_element(struct i40e_cloud_filter *filter,
6864                      struct i40e_aqc_cloud_filters_element_data *cld)
6865 {
6866         int i, j;
6867         u32 ipa;
6868
6869         memset(cld, 0, sizeof(*cld));
6870         ether_addr_copy(cld->outer_mac, filter->dst_mac);
6871         ether_addr_copy(cld->inner_mac, filter->src_mac);
6872
6873         if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
6874                 return;
6875
6876         if (filter->n_proto == ETH_P_IPV6) {
6877 #define IPV6_MAX_INDEX  (ARRAY_SIZE(filter->dst_ipv6) - 1)
6878                 for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6);
6879                      i++, j += 2) {
6880                         ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
6881                         ipa = cpu_to_le32(ipa);
6882                         memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa));
6883                 }
6884         } else {
6885                 ipa = be32_to_cpu(filter->dst_ipv4);
6886                 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
6887         }
6888
6889         cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
6890
6891         /* tenant_id is not supported by FW now, once the support is enabled
6892          * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
6893          */
6894         if (filter->tenant_id)
6895                 return;
6896 }
6897
6898 /**
6899  * i40e_add_del_cloud_filter - Add/del cloud filter
6900  * @vsi: pointer to VSI
6901  * @filter: cloud filter rule
6902  * @add: if true, add, if false, delete
6903  *
6904  * Add or delete a cloud filter for a specific flow spec.
6905  * Returns 0 if the filter were successfully added.
6906  **/
6907 static int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
6908                                      struct i40e_cloud_filter *filter, bool add)
6909 {
6910         struct i40e_aqc_cloud_filters_element_data cld_filter;
6911         struct i40e_pf *pf = vsi->back;
6912         int ret;
6913         static const u16 flag_table[128] = {
6914                 [I40E_CLOUD_FILTER_FLAGS_OMAC]  =
6915                         I40E_AQC_ADD_CLOUD_FILTER_OMAC,
6916                 [I40E_CLOUD_FILTER_FLAGS_IMAC]  =
6917                         I40E_AQC_ADD_CLOUD_FILTER_IMAC,
6918                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN]  =
6919                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
6920                 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
6921                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
6922                 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
6923                         I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
6924                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
6925                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
6926                 [I40E_CLOUD_FILTER_FLAGS_IIP] =
6927                         I40E_AQC_ADD_CLOUD_FILTER_IIP,
6928         };
6929
6930         if (filter->flags >= ARRAY_SIZE(flag_table))
6931                 return I40E_ERR_CONFIG;
6932
6933         /* copy element needed to add cloud filter from filter */
6934         i40e_set_cld_element(filter, &cld_filter);
6935
6936         if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
6937                 cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
6938                                              I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
6939
6940         if (filter->n_proto == ETH_P_IPV6)
6941                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
6942                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
6943         else
6944                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
6945                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
6946
6947         if (add)
6948                 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
6949                                                 &cld_filter, 1);
6950         else
6951                 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
6952                                                 &cld_filter, 1);
6953         if (ret)
6954                 dev_dbg(&pf->pdev->dev,
6955                         "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
6956                         add ? "add" : "delete", filter->dst_port, ret,
6957                         pf->hw.aq.asq_last_status);
6958         else
6959                 dev_info(&pf->pdev->dev,
6960                          "%s cloud filter for VSI: %d\n",
6961                          add ? "Added" : "Deleted", filter->seid);
6962         return ret;
6963 }
6964
6965 /**
6966  * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
6967  * @vsi: pointer to VSI
6968  * @filter: cloud filter rule
6969  * @add: if true, add, if false, delete
6970  *
6971  * Add or delete a cloud filter for a specific flow spec using big buffer.
6972  * Returns 0 if the filter were successfully added.
6973  **/
6974 static int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
6975                                              struct i40e_cloud_filter *filter,
6976                                              bool add)
6977 {
6978         struct i40e_aqc_cloud_filters_element_bb cld_filter;
6979         struct i40e_pf *pf = vsi->back;
6980         int ret;
6981
6982         /* Both (src/dst) valid mac_addr are not supported */
6983         if ((is_valid_ether_addr(filter->dst_mac) &&
6984              is_valid_ether_addr(filter->src_mac)) ||
6985             (is_multicast_ether_addr(filter->dst_mac) &&
6986              is_multicast_ether_addr(filter->src_mac)))
6987                 return -EOPNOTSUPP;
6988
6989         /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
6990          * ports are not supported via big buffer now.
6991          */
6992         if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
6993                 return -EOPNOTSUPP;
6994
6995         /* adding filter using src_port/src_ip is not supported at this stage */
6996         if (filter->src_port || filter->src_ipv4 ||
6997             !ipv6_addr_any(&filter->ip.v6.src_ip6))
6998                 return -EOPNOTSUPP;
6999
7000         /* copy element needed to add cloud filter from filter */
7001         i40e_set_cld_element(filter, &cld_filter.element);
7002
7003         if (is_valid_ether_addr(filter->dst_mac) ||
7004             is_valid_ether_addr(filter->src_mac) ||
7005             is_multicast_ether_addr(filter->dst_mac) ||
7006             is_multicast_ether_addr(filter->src_mac)) {
7007                 /* MAC + IP : unsupported mode */
7008                 if (filter->dst_ipv4)
7009                         return -EOPNOTSUPP;
7010
7011                 /* since we validated that L4 port must be valid before
7012                  * we get here, start with respective "flags" value
7013                  * and update if vlan is present or not
7014                  */
7015                 cld_filter.element.flags =
7016                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
7017
7018                 if (filter->vlan_id) {
7019                         cld_filter.element.flags =
7020                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
7021                 }
7022
7023         } else if (filter->dst_ipv4 ||
7024                    !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
7025                 cld_filter.element.flags =
7026                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
7027                 if (filter->n_proto == ETH_P_IPV6)
7028                         cld_filter.element.flags |=
7029                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
7030                 else
7031                         cld_filter.element.flags |=
7032                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
7033         } else {
7034                 dev_err(&pf->pdev->dev,
7035                         "either mac or ip has to be valid for cloud filter\n");
7036                 return -EINVAL;
7037         }
7038
7039         /* Now copy L4 port in Byte 6..7 in general fields */
7040         cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
7041                                                 be16_to_cpu(filter->dst_port);
7042
7043         if (add) {
7044                 /* Validate current device switch mode, change if necessary */
7045                 ret = i40e_validate_and_set_switch_mode(vsi);
7046                 if (ret) {
7047                         dev_err(&pf->pdev->dev,
7048                                 "failed to set switch mode, ret %d\n",
7049                                 ret);
7050                         return ret;
7051                 }
7052
7053                 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
7054                                                    &cld_filter, 1);
7055         } else {
7056                 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
7057                                                    &cld_filter, 1);
7058         }
7059
7060         if (ret)
7061                 dev_dbg(&pf->pdev->dev,
7062                         "Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
7063                         add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
7064         else
7065                 dev_info(&pf->pdev->dev,
7066                          "%s cloud filter for VSI: %d, L4 port: %d\n",
7067                          add ? "add" : "delete", filter->seid,
7068                          ntohs(filter->dst_port));
7069         return ret;
7070 }
7071
7072 /**
7073  * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
7074  * @vsi: Pointer to VSI
7075  * @cls_flower: Pointer to struct tc_cls_flower_offload
7076  * @filter: Pointer to cloud filter structure
7077  *
7078  **/
7079 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
7080                                  struct tc_cls_flower_offload *f,
7081                                  struct i40e_cloud_filter *filter)
7082 {
7083         u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
7084         struct i40e_pf *pf = vsi->back;
7085         u8 field_flags = 0;
7086
7087         if (f->dissector->used_keys &
7088             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
7089               BIT(FLOW_DISSECTOR_KEY_BASIC) |
7090               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
7091               BIT(FLOW_DISSECTOR_KEY_VLAN) |
7092               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
7093               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
7094               BIT(FLOW_DISSECTOR_KEY_PORTS) |
7095               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
7096                 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n",
7097                         f->dissector->used_keys);
7098                 return -EOPNOTSUPP;
7099         }
7100
7101         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
7102                 struct flow_dissector_key_keyid *key =
7103                         skb_flow_dissector_target(f->dissector,
7104                                                   FLOW_DISSECTOR_KEY_ENC_KEYID,
7105                                                   f->key);
7106
7107                 struct flow_dissector_key_keyid *mask =
7108                         skb_flow_dissector_target(f->dissector,
7109                                                   FLOW_DISSECTOR_KEY_ENC_KEYID,
7110                                                   f->mask);
7111
7112                 if (mask->keyid != 0)
7113                         field_flags |= I40E_CLOUD_FIELD_TEN_ID;
7114
7115                 filter->tenant_id = be32_to_cpu(key->keyid);
7116         }
7117
7118         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
7119                 struct flow_dissector_key_basic *key =
7120                         skb_flow_dissector_target(f->dissector,
7121                                                   FLOW_DISSECTOR_KEY_BASIC,
7122                                                   f->key);
7123
7124                 struct flow_dissector_key_basic *mask =
7125                         skb_flow_dissector_target(f->dissector,
7126                                                   FLOW_DISSECTOR_KEY_BASIC,
7127                                                   f->mask);
7128
7129                 n_proto_key = ntohs(key->n_proto);
7130                 n_proto_mask = ntohs(mask->n_proto);
7131
7132                 if (n_proto_key == ETH_P_ALL) {
7133                         n_proto_key = 0;
7134                         n_proto_mask = 0;
7135                 }
7136                 filter->n_proto = n_proto_key & n_proto_mask;
7137                 filter->ip_proto = key->ip_proto;
7138         }
7139
7140         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
7141                 struct flow_dissector_key_eth_addrs *key =
7142                         skb_flow_dissector_target(f->dissector,
7143                                                   FLOW_DISSECTOR_KEY_ETH_ADDRS,
7144                                                   f->key);
7145
7146                 struct flow_dissector_key_eth_addrs *mask =
7147                         skb_flow_dissector_target(f->dissector,
7148                                                   FLOW_DISSECTOR_KEY_ETH_ADDRS,
7149                                                   f->mask);
7150
7151                 /* use is_broadcast and is_zero to check for all 0xf or 0 */
7152                 if (!is_zero_ether_addr(mask->dst)) {
7153                         if (is_broadcast_ether_addr(mask->dst)) {
7154                                 field_flags |= I40E_CLOUD_FIELD_OMAC;
7155                         } else {
7156                                 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
7157                                         mask->dst);
7158                                 return I40E_ERR_CONFIG;
7159                         }
7160                 }
7161
7162                 if (!is_zero_ether_addr(mask->src)) {
7163                         if (is_broadcast_ether_addr(mask->src)) {
7164                                 field_flags |= I40E_CLOUD_FIELD_IMAC;
7165                         } else {
7166                                 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
7167                                         mask->src);
7168                                 return I40E_ERR_CONFIG;
7169                         }
7170                 }
7171                 ether_addr_copy(filter->dst_mac, key->dst);
7172                 ether_addr_copy(filter->src_mac, key->src);
7173         }
7174
7175         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
7176                 struct flow_dissector_key_vlan *key =
7177                         skb_flow_dissector_target(f->dissector,
7178                                                   FLOW_DISSECTOR_KEY_VLAN,
7179                                                   f->key);
7180                 struct flow_dissector_key_vlan *mask =
7181                         skb_flow_dissector_target(f->dissector,
7182                                                   FLOW_DISSECTOR_KEY_VLAN,
7183                                                   f->mask);
7184
7185                 if (mask->vlan_id) {
7186                         if (mask->vlan_id == VLAN_VID_MASK) {
7187                                 field_flags |= I40E_CLOUD_FIELD_IVLAN;
7188
7189                         } else {
7190                                 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
7191                                         mask->vlan_id);
7192                                 return I40E_ERR_CONFIG;
7193                         }
7194                 }
7195
7196                 filter->vlan_id = cpu_to_be16(key->vlan_id);
7197         }
7198
7199         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
7200                 struct flow_dissector_key_control *key =
7201                         skb_flow_dissector_target(f->dissector,
7202                                                   FLOW_DISSECTOR_KEY_CONTROL,
7203                                                   f->key);
7204
7205                 addr_type = key->addr_type;
7206         }
7207
7208         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
7209                 struct flow_dissector_key_ipv4_addrs *key =
7210                         skb_flow_dissector_target(f->dissector,
7211                                                   FLOW_DISSECTOR_KEY_IPV4_ADDRS,
7212                                                   f->key);
7213                 struct flow_dissector_key_ipv4_addrs *mask =
7214                         skb_flow_dissector_target(f->dissector,
7215                                                   FLOW_DISSECTOR_KEY_IPV4_ADDRS,
7216                                                   f->mask);
7217
7218                 if (mask->dst) {
7219                         if (mask->dst == cpu_to_be32(0xffffffff)) {
7220                                 field_flags |= I40E_CLOUD_FIELD_IIP;
7221                         } else {
7222                                 mask->dst = be32_to_cpu(mask->dst);
7223                                 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4\n",
7224                                         &mask->dst);
7225                                 return I40E_ERR_CONFIG;
7226                         }
7227                 }
7228
7229                 if (mask->src) {
7230                         if (mask->src == cpu_to_be32(0xffffffff)) {
7231                                 field_flags |= I40E_CLOUD_FIELD_IIP;
7232                         } else {
7233                                 mask->src = be32_to_cpu(mask->src);
7234                                 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4\n",
7235                                         &mask->src);
7236                                 return I40E_ERR_CONFIG;
7237                         }
7238                 }
7239
7240                 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
7241                         dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
7242                         return I40E_ERR_CONFIG;
7243                 }
7244                 filter->dst_ipv4 = key->dst;
7245                 filter->src_ipv4 = key->src;
7246         }
7247
7248         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
7249                 struct flow_dissector_key_ipv6_addrs *key =
7250                         skb_flow_dissector_target(f->dissector,
7251                                                   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
7252                                                   f->key);
7253                 struct flow_dissector_key_ipv6_addrs *mask =
7254                         skb_flow_dissector_target(f->dissector,
7255                                                   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
7256                                                   f->mask);
7257
7258                 /* src and dest IPV6 address should not be LOOPBACK
7259                  * (0:0:0:0:0:0:0:1), which can be represented as ::1
7260                  */
7261                 if (ipv6_addr_loopback(&key->dst) ||
7262                     ipv6_addr_loopback(&key->src)) {
7263                         dev_err(&pf->pdev->dev,
7264                                 "Bad ipv6, addr is LOOPBACK\n");
7265                         return I40E_ERR_CONFIG;
7266                 }
7267                 if (!ipv6_addr_any(&mask->dst) || !ipv6_addr_any(&mask->src))
7268                         field_flags |= I40E_CLOUD_FIELD_IIP;
7269
7270                 memcpy(&filter->src_ipv6, &key->src.s6_addr32,
7271                        sizeof(filter->src_ipv6));
7272                 memcpy(&filter->dst_ipv6, &key->dst.s6_addr32,
7273                        sizeof(filter->dst_ipv6));
7274         }
7275
7276         if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) {
7277                 struct flow_dissector_key_ports *key =
7278                         skb_flow_dissector_target(f->dissector,
7279                                                   FLOW_DISSECTOR_KEY_PORTS,
7280                                                   f->key);
7281                 struct flow_dissector_key_ports *mask =
7282                         skb_flow_dissector_target(f->dissector,
7283                                                   FLOW_DISSECTOR_KEY_PORTS,
7284                                                   f->mask);
7285
7286                 if (mask->src) {
7287                         if (mask->src == cpu_to_be16(0xffff)) {
7288                                 field_flags |= I40E_CLOUD_FIELD_IIP;
7289                         } else {
7290                                 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
7291                                         be16_to_cpu(mask->src));
7292                                 return I40E_ERR_CONFIG;
7293                         }
7294                 }
7295
7296                 if (mask->dst) {
7297                         if (mask->dst == cpu_to_be16(0xffff)) {
7298                                 field_flags |= I40E_CLOUD_FIELD_IIP;
7299                         } else {
7300                                 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
7301                                         be16_to_cpu(mask->dst));
7302                                 return I40E_ERR_CONFIG;
7303                         }
7304                 }
7305
7306                 filter->dst_port = key->dst;
7307                 filter->src_port = key->src;
7308
7309                 switch (filter->ip_proto) {
7310                 case IPPROTO_TCP:
7311                 case IPPROTO_UDP:
7312                         break;
7313                 default:
7314                         dev_err(&pf->pdev->dev,
7315                                 "Only UDP and TCP transport are supported\n");
7316                         return -EINVAL;
7317                 }
7318         }
7319         filter->flags = field_flags;
7320         return 0;
7321 }
7322
7323 /**
7324  * i40e_handle_tclass: Forward to a traffic class on the device
7325  * @vsi: Pointer to VSI
7326  * @tc: traffic class index on the device
7327  * @filter: Pointer to cloud filter structure
7328  *
7329  **/
7330 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
7331                               struct i40e_cloud_filter *filter)
7332 {
7333         struct i40e_channel *ch, *ch_tmp;
7334
7335         /* direct to a traffic class on the same device */
7336         if (tc == 0) {
7337                 filter->seid = vsi->seid;
7338                 return 0;
7339         } else if (vsi->tc_config.enabled_tc & BIT(tc)) {
7340                 if (!filter->dst_port) {
7341                         dev_err(&vsi->back->pdev->dev,
7342                                 "Specify destination port to direct to traffic class that is not default\n");
7343                         return -EINVAL;
7344                 }
7345                 if (list_empty(&vsi->ch_list))
7346                         return -EINVAL;
7347                 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
7348                                          list) {
7349                         if (ch->seid == vsi->tc_seid_map[tc])
7350                                 filter->seid = ch->seid;
7351                 }
7352                 return 0;
7353         }
7354         dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
7355         return -EINVAL;
7356 }
7357
7358 /**
7359  * i40e_configure_clsflower - Configure tc flower filters
7360  * @vsi: Pointer to VSI
7361  * @cls_flower: Pointer to struct tc_cls_flower_offload
7362  *
7363  **/
7364 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
7365                                     struct tc_cls_flower_offload *cls_flower)
7366 {
7367         int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
7368         struct i40e_cloud_filter *filter = NULL;
7369         struct i40e_pf *pf = vsi->back;
7370         int err = 0;
7371
7372         if (tc < 0) {
7373                 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
7374                 return -EOPNOTSUPP;
7375         }
7376
7377         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
7378             test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
7379                 return -EBUSY;
7380
7381         if (pf->fdir_pf_active_filters ||
7382             (!hlist_empty(&pf->fdir_filter_list))) {
7383                 dev_err(&vsi->back->pdev->dev,
7384                         "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
7385                 return -EINVAL;
7386         }
7387
7388         if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) {
7389                 dev_err(&vsi->back->pdev->dev,
7390                         "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
7391                 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED;
7392                 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
7393         }
7394
7395         filter = kzalloc(sizeof(*filter), GFP_KERNEL);
7396         if (!filter)
7397                 return -ENOMEM;
7398
7399         filter->cookie = cls_flower->cookie;
7400
7401         err = i40e_parse_cls_flower(vsi, cls_flower, filter);
7402         if (err < 0)
7403                 goto err;
7404
7405         err = i40e_handle_tclass(vsi, tc, filter);
7406         if (err < 0)
7407                 goto err;
7408
7409         /* Add cloud filter */
7410         if (filter->dst_port)
7411                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
7412         else
7413                 err = i40e_add_del_cloud_filter(vsi, filter, true);
7414
7415         if (err) {
7416                 dev_err(&pf->pdev->dev,
7417                         "Failed to add cloud filter, err %s\n",
7418                         i40e_stat_str(&pf->hw, err));
7419                 goto err;
7420         }
7421
7422         /* add filter to the ordered list */
7423         INIT_HLIST_NODE(&filter->cloud_node);
7424
7425         hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
7426
7427         pf->num_cloud_filters++;
7428
7429         return err;
7430 err:
7431         kfree(filter);
7432         return err;
7433 }
7434
7435 /**
7436  * i40e_find_cloud_filter - Find the could filter in the list
7437  * @vsi: Pointer to VSI
7438  * @cookie: filter specific cookie
7439  *
7440  **/
7441 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
7442                                                         unsigned long *cookie)
7443 {
7444         struct i40e_cloud_filter *filter = NULL;
7445         struct hlist_node *node2;
7446
7447         hlist_for_each_entry_safe(filter, node2,
7448                                   &vsi->back->cloud_filter_list, cloud_node)
7449                 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
7450                         return filter;
7451         return NULL;
7452 }
7453
7454 /**
7455  * i40e_delete_clsflower - Remove tc flower filters
7456  * @vsi: Pointer to VSI
7457  * @cls_flower: Pointer to struct tc_cls_flower_offload
7458  *
7459  **/
7460 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
7461                                  struct tc_cls_flower_offload *cls_flower)
7462 {
7463         struct i40e_cloud_filter *filter = NULL;
7464         struct i40e_pf *pf = vsi->back;
7465         int err = 0;
7466
7467         filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
7468
7469         if (!filter)
7470                 return -EINVAL;
7471
7472         hash_del(&filter->cloud_node);
7473
7474         if (filter->dst_port)
7475                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
7476         else
7477                 err = i40e_add_del_cloud_filter(vsi, filter, false);
7478
7479         kfree(filter);
7480         if (err) {
7481                 dev_err(&pf->pdev->dev,
7482                         "Failed to delete cloud filter, err %s\n",
7483                         i40e_stat_str(&pf->hw, err));
7484                 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
7485         }
7486
7487         pf->num_cloud_filters--;
7488         if (!pf->num_cloud_filters)
7489                 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
7490                     !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
7491                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
7492                         pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
7493                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
7494                 }
7495         return 0;
7496 }
7497
7498 /**
7499  * i40e_setup_tc_cls_flower - flower classifier offloads
7500  * @netdev: net device to configure
7501  * @type_data: offload data
7502  **/
7503 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
7504                                     struct tc_cls_flower_offload *cls_flower)
7505 {
7506         struct i40e_vsi *vsi = np->vsi;
7507
7508         if (!tc_can_offload(vsi->netdev))
7509                 return -EOPNOTSUPP;
7510         if (cls_flower->common.chain_index)
7511                 return -EOPNOTSUPP;
7512
7513         switch (cls_flower->command) {
7514         case TC_CLSFLOWER_REPLACE:
7515                 return i40e_configure_clsflower(vsi, cls_flower);
7516         case TC_CLSFLOWER_DESTROY:
7517                 return i40e_delete_clsflower(vsi, cls_flower);
7518         case TC_CLSFLOWER_STATS:
7519                 return -EOPNOTSUPP;
7520         default:
7521                 return -EINVAL;
7522         }
7523 }
7524
7525 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
7526                                   void *cb_priv)
7527 {
7528         struct i40e_netdev_priv *np = cb_priv;
7529
7530         switch (type) {
7531         case TC_SETUP_CLSFLOWER:
7532                 return i40e_setup_tc_cls_flower(np, type_data);
7533
7534         default:
7535                 return -EOPNOTSUPP;
7536         }
7537 }
7538
7539 static int i40e_setup_tc_block(struct net_device *dev,
7540                                struct tc_block_offload *f)
7541 {
7542         struct i40e_netdev_priv *np = netdev_priv(dev);
7543
7544         if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
7545                 return -EOPNOTSUPP;
7546
7547         switch (f->command) {
7548         case TC_BLOCK_BIND:
7549                 return tcf_block_cb_register(f->block, i40e_setup_tc_block_cb,
7550                                              np, np);
7551         case TC_BLOCK_UNBIND:
7552                 tcf_block_cb_unregister(f->block, i40e_setup_tc_block_cb, np);
7553                 return 0;
7554         default:
7555                 return -EOPNOTSUPP;
7556         }
7557 }
7558
7559 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
7560                            void *type_data)
7561 {
7562         switch (type) {
7563         case TC_SETUP_QDISC_MQPRIO:
7564                 return i40e_setup_tc(netdev, type_data);
7565         case TC_SETUP_BLOCK:
7566                 return i40e_setup_tc_block(netdev, type_data);
7567         default:
7568                 return -EOPNOTSUPP;
7569         }
7570 }
7571
7572 /**
7573  * i40e_open - Called when a network interface is made active
7574  * @netdev: network interface device structure
7575  *
7576  * The open entry point is called when a network interface is made
7577  * active by the system (IFF_UP).  At this point all resources needed
7578  * for transmit and receive operations are allocated, the interrupt
7579  * handler is registered with the OS, the netdev watchdog subtask is
7580  * enabled, and the stack is notified that the interface is ready.
7581  *
7582  * Returns 0 on success, negative value on failure
7583  **/
7584 int i40e_open(struct net_device *netdev)
7585 {
7586         struct i40e_netdev_priv *np = netdev_priv(netdev);
7587         struct i40e_vsi *vsi = np->vsi;
7588         struct i40e_pf *pf = vsi->back;
7589         int err;
7590
7591         /* disallow open during test or if eeprom is broken */
7592         if (test_bit(__I40E_TESTING, pf->state) ||
7593             test_bit(__I40E_BAD_EEPROM, pf->state))
7594                 return -EBUSY;
7595
7596         netif_carrier_off(netdev);
7597
7598         err = i40e_vsi_open(vsi);
7599         if (err)
7600                 return err;
7601
7602         /* configure global TSO hardware offload settings */
7603         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
7604                                                        TCP_FLAG_FIN) >> 16);
7605         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
7606                                                        TCP_FLAG_FIN |
7607                                                        TCP_FLAG_CWR) >> 16);
7608         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
7609
7610         udp_tunnel_get_rx_info(netdev);
7611
7612         return 0;
7613 }
7614
7615 /**
7616  * i40e_vsi_open -
7617  * @vsi: the VSI to open
7618  *
7619  * Finish initialization of the VSI.
7620  *
7621  * Returns 0 on success, negative value on failure
7622  *
7623  * Note: expects to be called while under rtnl_lock()
7624  **/
7625 int i40e_vsi_open(struct i40e_vsi *vsi)
7626 {
7627         struct i40e_pf *pf = vsi->back;
7628         char int_name[I40E_INT_NAME_STR_LEN];
7629         int err;
7630
7631         /* allocate descriptors */
7632         err = i40e_vsi_setup_tx_resources(vsi);
7633         if (err)
7634                 goto err_setup_tx;
7635         err = i40e_vsi_setup_rx_resources(vsi);
7636         if (err)
7637                 goto err_setup_rx;
7638
7639         err = i40e_vsi_configure(vsi);
7640         if (err)
7641                 goto err_setup_rx;
7642
7643         if (vsi->netdev) {
7644                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
7645                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
7646                 err = i40e_vsi_request_irq(vsi, int_name);
7647                 if (err)
7648                         goto err_setup_rx;
7649
7650                 /* Notify the stack of the actual queue counts. */
7651                 err = netif_set_real_num_tx_queues(vsi->netdev,
7652                                                    vsi->num_queue_pairs);
7653                 if (err)
7654                         goto err_set_queues;
7655
7656                 err = netif_set_real_num_rx_queues(vsi->netdev,
7657                                                    vsi->num_queue_pairs);
7658                 if (err)
7659                         goto err_set_queues;
7660
7661         } else if (vsi->type == I40E_VSI_FDIR) {
7662                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
7663                          dev_driver_string(&pf->pdev->dev),
7664                          dev_name(&pf->pdev->dev));
7665                 err = i40e_vsi_request_irq(vsi, int_name);
7666
7667         } else {
7668                 err = -EINVAL;
7669                 goto err_setup_rx;
7670         }
7671
7672         err = i40e_up_complete(vsi);
7673         if (err)
7674                 goto err_up_complete;
7675
7676         return 0;
7677
7678 err_up_complete:
7679         i40e_down(vsi);
7680 err_set_queues:
7681         i40e_vsi_free_irq(vsi);
7682 err_setup_rx:
7683         i40e_vsi_free_rx_resources(vsi);
7684 err_setup_tx:
7685         i40e_vsi_free_tx_resources(vsi);
7686         if (vsi == pf->vsi[pf->lan_vsi])
7687                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
7688
7689         return err;
7690 }
7691
7692 /**
7693  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
7694  * @pf: Pointer to PF
7695  *
7696  * This function destroys the hlist where all the Flow Director
7697  * filters were saved.
7698  **/
7699 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
7700 {
7701         struct i40e_fdir_filter *filter;
7702         struct i40e_flex_pit *pit_entry, *tmp;
7703         struct hlist_node *node2;
7704
7705         hlist_for_each_entry_safe(filter, node2,
7706                                   &pf->fdir_filter_list, fdir_node) {
7707                 hlist_del(&filter->fdir_node);
7708                 kfree(filter);
7709         }
7710
7711         list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
7712                 list_del(&pit_entry->list);
7713                 kfree(pit_entry);
7714         }
7715         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
7716
7717         list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
7718                 list_del(&pit_entry->list);
7719                 kfree(pit_entry);
7720         }
7721         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
7722
7723         pf->fdir_pf_active_filters = 0;
7724         pf->fd_tcp4_filter_cnt = 0;
7725         pf->fd_udp4_filter_cnt = 0;
7726         pf->fd_sctp4_filter_cnt = 0;
7727         pf->fd_ip4_filter_cnt = 0;
7728
7729         /* Reprogram the default input set for TCP/IPv4 */
7730         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
7731                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
7732                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
7733
7734         /* Reprogram the default input set for UDP/IPv4 */
7735         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
7736                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
7737                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
7738
7739         /* Reprogram the default input set for SCTP/IPv4 */
7740         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
7741                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
7742                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
7743
7744         /* Reprogram the default input set for Other/IPv4 */
7745         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
7746                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
7747 }
7748
7749 /**
7750  * i40e_cloud_filter_exit - Cleans up the cloud filters
7751  * @pf: Pointer to PF
7752  *
7753  * This function destroys the hlist where all the cloud filters
7754  * were saved.
7755  **/
7756 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
7757 {
7758         struct i40e_cloud_filter *cfilter;
7759         struct hlist_node *node;
7760
7761         hlist_for_each_entry_safe(cfilter, node,
7762                                   &pf->cloud_filter_list, cloud_node) {
7763                 hlist_del(&cfilter->cloud_node);
7764                 kfree(cfilter);
7765         }
7766         pf->num_cloud_filters = 0;
7767
7768         if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
7769             !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
7770                 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
7771                 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
7772                 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
7773         }
7774 }
7775
7776 /**
7777  * i40e_close - Disables a network interface
7778  * @netdev: network interface device structure
7779  *
7780  * The close entry point is called when an interface is de-activated
7781  * by the OS.  The hardware is still under the driver's control, but
7782  * this netdev interface is disabled.
7783  *
7784  * Returns 0, this is not allowed to fail
7785  **/
7786 int i40e_close(struct net_device *netdev)
7787 {
7788         struct i40e_netdev_priv *np = netdev_priv(netdev);
7789         struct i40e_vsi *vsi = np->vsi;
7790
7791         i40e_vsi_close(vsi);
7792
7793         return 0;
7794 }
7795
7796 /**
7797  * i40e_do_reset - Start a PF or Core Reset sequence
7798  * @pf: board private structure
7799  * @reset_flags: which reset is requested
7800  * @lock_acquired: indicates whether or not the lock has been acquired
7801  * before this function was called.
7802  *
7803  * The essential difference in resets is that the PF Reset
7804  * doesn't clear the packet buffers, doesn't reset the PE
7805  * firmware, and doesn't bother the other PFs on the chip.
7806  **/
7807 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
7808 {
7809         u32 val;
7810
7811         WARN_ON(in_interrupt());
7812
7813
7814         /* do the biggest reset indicated */
7815         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
7816
7817                 /* Request a Global Reset
7818                  *
7819                  * This will start the chip's countdown to the actual full
7820                  * chip reset event, and a warning interrupt to be sent
7821                  * to all PFs, including the requestor.  Our handler
7822                  * for the warning interrupt will deal with the shutdown
7823                  * and recovery of the switch setup.
7824                  */
7825                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
7826                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
7827                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
7828                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
7829
7830         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
7831
7832                 /* Request a Core Reset
7833                  *
7834                  * Same as Global Reset, except does *not* include the MAC/PHY
7835                  */
7836                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
7837                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
7838                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
7839                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
7840                 i40e_flush(&pf->hw);
7841
7842         } else if (reset_flags & I40E_PF_RESET_FLAG) {
7843
7844                 /* Request a PF Reset
7845                  *
7846                  * Resets only the PF-specific registers
7847                  *
7848                  * This goes directly to the tear-down and rebuild of
7849                  * the switch, since we need to do all the recovery as
7850                  * for the Core Reset.
7851                  */
7852                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
7853                 i40e_handle_reset_warning(pf, lock_acquired);
7854
7855         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
7856                 int v;
7857
7858                 /* Find the VSI(s) that requested a re-init */
7859                 dev_info(&pf->pdev->dev,
7860                          "VSI reinit requested\n");
7861                 for (v = 0; v < pf->num_alloc_vsi; v++) {
7862                         struct i40e_vsi *vsi = pf->vsi[v];
7863
7864                         if (vsi != NULL &&
7865                             test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
7866                                                vsi->state))
7867                                 i40e_vsi_reinit_locked(pf->vsi[v]);
7868                 }
7869         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
7870                 int v;
7871
7872                 /* Find the VSI(s) that needs to be brought down */
7873                 dev_info(&pf->pdev->dev, "VSI down requested\n");
7874                 for (v = 0; v < pf->num_alloc_vsi; v++) {
7875                         struct i40e_vsi *vsi = pf->vsi[v];
7876
7877                         if (vsi != NULL &&
7878                             test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
7879                                                vsi->state)) {
7880                                 set_bit(__I40E_VSI_DOWN, vsi->state);
7881                                 i40e_down(vsi);
7882                         }
7883                 }
7884         } else {
7885                 dev_info(&pf->pdev->dev,
7886                          "bad reset request 0x%08x\n", reset_flags);
7887         }
7888 }
7889
7890 #ifdef CONFIG_I40E_DCB
7891 /**
7892  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
7893  * @pf: board private structure
7894  * @old_cfg: current DCB config
7895  * @new_cfg: new DCB config
7896  **/
7897 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
7898                             struct i40e_dcbx_config *old_cfg,
7899                             struct i40e_dcbx_config *new_cfg)
7900 {
7901         bool need_reconfig = false;
7902
7903         /* Check if ETS configuration has changed */
7904         if (memcmp(&new_cfg->etscfg,
7905                    &old_cfg->etscfg,
7906                    sizeof(new_cfg->etscfg))) {
7907                 /* If Priority Table has changed reconfig is needed */
7908                 if (memcmp(&new_cfg->etscfg.prioritytable,
7909                            &old_cfg->etscfg.prioritytable,
7910                            sizeof(new_cfg->etscfg.prioritytable))) {
7911                         need_reconfig = true;
7912                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
7913                 }
7914
7915                 if (memcmp(&new_cfg->etscfg.tcbwtable,
7916                            &old_cfg->etscfg.tcbwtable,
7917                            sizeof(new_cfg->etscfg.tcbwtable)))
7918                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
7919
7920                 if (memcmp(&new_cfg->etscfg.tsatable,
7921                            &old_cfg->etscfg.tsatable,
7922                            sizeof(new_cfg->etscfg.tsatable)))
7923                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
7924         }
7925
7926         /* Check if PFC configuration has changed */
7927         if (memcmp(&new_cfg->pfc,
7928                    &old_cfg->pfc,
7929                    sizeof(new_cfg->pfc))) {
7930                 need_reconfig = true;
7931                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
7932         }
7933
7934         /* Check if APP Table has changed */
7935         if (memcmp(&new_cfg->app,
7936                    &old_cfg->app,
7937                    sizeof(new_cfg->app))) {
7938                 need_reconfig = true;
7939                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
7940         }
7941
7942         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
7943         return need_reconfig;
7944 }
7945
7946 /**
7947  * i40e_handle_lldp_event - Handle LLDP Change MIB event
7948  * @pf: board private structure
7949  * @e: event info posted on ARQ
7950  **/
7951 static int i40e_handle_lldp_event(struct i40e_pf *pf,
7952                                   struct i40e_arq_event_info *e)
7953 {
7954         struct i40e_aqc_lldp_get_mib *mib =
7955                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
7956         struct i40e_hw *hw = &pf->hw;
7957         struct i40e_dcbx_config tmp_dcbx_cfg;
7958         bool need_reconfig = false;
7959         int ret = 0;
7960         u8 type;
7961
7962         /* Not DCB capable or capability disabled */
7963         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
7964                 return ret;
7965
7966         /* Ignore if event is not for Nearest Bridge */
7967         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
7968                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
7969         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
7970         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
7971                 return ret;
7972
7973         /* Check MIB Type and return if event for Remote MIB update */
7974         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
7975         dev_dbg(&pf->pdev->dev,
7976                 "LLDP event mib type %s\n", type ? "remote" : "local");
7977         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
7978                 /* Update the remote cached instance and return */
7979                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
7980                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
7981                                 &hw->remote_dcbx_config);
7982                 goto exit;
7983         }
7984
7985         /* Store the old configuration */
7986         tmp_dcbx_cfg = hw->local_dcbx_config;
7987
7988         /* Reset the old DCBx configuration data */
7989         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
7990         /* Get updated DCBX data from firmware */
7991         ret = i40e_get_dcb_config(&pf->hw);
7992         if (ret) {
7993                 dev_info(&pf->pdev->dev,
7994                          "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
7995                          i40e_stat_str(&pf->hw, ret),
7996                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7997                 goto exit;
7998         }
7999
8000         /* No change detected in DCBX configs */
8001         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
8002                     sizeof(tmp_dcbx_cfg))) {
8003                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
8004                 goto exit;
8005         }
8006
8007         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
8008                                                &hw->local_dcbx_config);
8009
8010         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
8011
8012         if (!need_reconfig)
8013                 goto exit;
8014
8015         /* Enable DCB tagging only when more than one TC */
8016         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
8017                 pf->flags |= I40E_FLAG_DCB_ENABLED;
8018         else
8019                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
8020
8021         set_bit(__I40E_PORT_SUSPENDED, pf->state);
8022         /* Reconfiguration needed quiesce all VSIs */
8023         i40e_pf_quiesce_all_vsi(pf);
8024
8025         /* Changes in configuration update VEB/VSI */
8026         i40e_dcb_reconfigure(pf);
8027
8028         ret = i40e_resume_port_tx(pf);
8029
8030         clear_bit(__I40E_PORT_SUSPENDED, pf->state);
8031         /* In case of error no point in resuming VSIs */
8032         if (ret)
8033                 goto exit;
8034
8035         /* Wait for the PF's queues to be disabled */
8036         ret = i40e_pf_wait_queues_disabled(pf);
8037         if (ret) {
8038                 /* Schedule PF reset to recover */
8039                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
8040                 i40e_service_event_schedule(pf);
8041         } else {
8042                 i40e_pf_unquiesce_all_vsi(pf);
8043         pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED |
8044                       I40E_FLAG_CLIENT_L2_CHANGE);
8045         }
8046
8047 exit:
8048         return ret;
8049 }
8050 #endif /* CONFIG_I40E_DCB */
8051
8052 /**
8053  * i40e_do_reset_safe - Protected reset path for userland calls.
8054  * @pf: board private structure
8055  * @reset_flags: which reset is requested
8056  *
8057  **/
8058 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
8059 {
8060         rtnl_lock();
8061         i40e_do_reset(pf, reset_flags, true);
8062         rtnl_unlock();
8063 }
8064
8065 /**
8066  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
8067  * @pf: board private structure
8068  * @e: event info posted on ARQ
8069  *
8070  * Handler for LAN Queue Overflow Event generated by the firmware for PF
8071  * and VF queues
8072  **/
8073 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
8074                                            struct i40e_arq_event_info *e)
8075 {
8076         struct i40e_aqc_lan_overflow *data =
8077                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
8078         u32 queue = le32_to_cpu(data->prtdcb_rupto);
8079         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
8080         struct i40e_hw *hw = &pf->hw;
8081         struct i40e_vf *vf;
8082         u16 vf_id;
8083
8084         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
8085                 queue, qtx_ctl);
8086
8087         /* Queue belongs to VF, find the VF and issue VF reset */
8088         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
8089             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
8090                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
8091                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
8092                 vf_id -= hw->func_caps.vf_base_id;
8093                 vf = &pf->vf[vf_id];
8094                 i40e_vc_notify_vf_reset(vf);
8095                 /* Allow VF to process pending reset notification */
8096                 msleep(20);
8097                 i40e_reset_vf(vf, false);
8098         }
8099 }
8100
8101 /**
8102  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
8103  * @pf: board private structure
8104  **/
8105 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
8106 {
8107         u32 val, fcnt_prog;
8108
8109         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
8110         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
8111         return fcnt_prog;
8112 }
8113
8114 /**
8115  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
8116  * @pf: board private structure
8117  **/
8118 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
8119 {
8120         u32 val, fcnt_prog;
8121
8122         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
8123         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
8124                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
8125                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
8126         return fcnt_prog;
8127 }
8128
8129 /**
8130  * i40e_get_global_fd_count - Get total FD filters programmed on device
8131  * @pf: board private structure
8132  **/
8133 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
8134 {
8135         u32 val, fcnt_prog;
8136
8137         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
8138         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
8139                     ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
8140                      I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
8141         return fcnt_prog;
8142 }
8143
8144 /**
8145  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
8146  * @pf: board private structure
8147  **/
8148 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
8149 {
8150         struct i40e_fdir_filter *filter;
8151         u32 fcnt_prog, fcnt_avail;
8152         struct hlist_node *node;
8153
8154         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
8155                 return;
8156
8157         /* Check if we have enough room to re-enable FDir SB capability. */
8158         fcnt_prog = i40e_get_global_fd_count(pf);
8159         fcnt_avail = pf->fdir_pf_filter_count;
8160         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
8161             (pf->fd_add_err == 0) ||
8162             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) {
8163                 if (pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED) {
8164                         pf->flags &= ~I40E_FLAG_FD_SB_AUTO_DISABLED;
8165                         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
8166                             (I40E_DEBUG_FD & pf->hw.debug_mask))
8167                                 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
8168                 }
8169         }
8170
8171         /* We should wait for even more space before re-enabling ATR.
8172          * Additionally, we cannot enable ATR as long as we still have TCP SB
8173          * rules active.
8174          */
8175         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
8176             (pf->fd_tcp4_filter_cnt == 0)) {
8177                 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) {
8178                         pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
8179                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
8180                             (I40E_DEBUG_FD & pf->hw.debug_mask))
8181                                 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
8182                 }
8183         }
8184
8185         /* if hw had a problem adding a filter, delete it */
8186         if (pf->fd_inv > 0) {
8187                 hlist_for_each_entry_safe(filter, node,
8188                                           &pf->fdir_filter_list, fdir_node) {
8189                         if (filter->fd_id == pf->fd_inv) {
8190                                 hlist_del(&filter->fdir_node);
8191                                 kfree(filter);
8192                                 pf->fdir_pf_active_filters--;
8193                                 pf->fd_inv = 0;
8194                         }
8195                 }
8196         }
8197 }
8198
8199 #define I40E_MIN_FD_FLUSH_INTERVAL 10
8200 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
8201 /**
8202  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
8203  * @pf: board private structure
8204  **/
8205 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
8206 {
8207         unsigned long min_flush_time;
8208         int flush_wait_retry = 50;
8209         bool disable_atr = false;
8210         int fd_room;
8211         int reg;
8212
8213         if (!time_after(jiffies, pf->fd_flush_timestamp +
8214                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
8215                 return;
8216
8217         /* If the flush is happening too quick and we have mostly SB rules we
8218          * should not re-enable ATR for some time.
8219          */
8220         min_flush_time = pf->fd_flush_timestamp +
8221                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
8222         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
8223
8224         if (!(time_after(jiffies, min_flush_time)) &&
8225             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
8226                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
8227                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
8228                 disable_atr = true;
8229         }
8230
8231         pf->fd_flush_timestamp = jiffies;
8232         pf->flags |= I40E_FLAG_FD_ATR_AUTO_DISABLED;
8233         /* flush all filters */
8234         wr32(&pf->hw, I40E_PFQF_CTL_1,
8235              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
8236         i40e_flush(&pf->hw);
8237         pf->fd_flush_cnt++;
8238         pf->fd_add_err = 0;
8239         do {
8240                 /* Check FD flush status every 5-6msec */
8241                 usleep_range(5000, 6000);
8242                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
8243                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
8244                         break;
8245         } while (flush_wait_retry--);
8246         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
8247                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
8248         } else {
8249                 /* replay sideband filters */
8250                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
8251                 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
8252                         pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
8253                 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
8254                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
8255                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
8256         }
8257 }
8258
8259 /**
8260  * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
8261  * @pf: board private structure
8262  **/
8263 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
8264 {
8265         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
8266 }
8267
8268 /* We can see up to 256 filter programming desc in transit if the filters are
8269  * being applied really fast; before we see the first
8270  * filter miss error on Rx queue 0. Accumulating enough error messages before
8271  * reacting will make sure we don't cause flush too often.
8272  */
8273 #define I40E_MAX_FD_PROGRAM_ERROR 256
8274
8275 /**
8276  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
8277  * @pf: board private structure
8278  **/
8279 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
8280 {
8281
8282         /* if interface is down do nothing */
8283         if (test_bit(__I40E_DOWN, pf->state))
8284                 return;
8285
8286         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
8287                 i40e_fdir_flush_and_replay(pf);
8288
8289         i40e_fdir_check_and_reenable(pf);
8290
8291 }
8292
8293 /**
8294  * i40e_vsi_link_event - notify VSI of a link event
8295  * @vsi: vsi to be notified
8296  * @link_up: link up or down
8297  **/
8298 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
8299 {
8300         if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
8301                 return;
8302
8303         switch (vsi->type) {
8304         case I40E_VSI_MAIN:
8305                 if (!vsi->netdev || !vsi->netdev_registered)
8306                         break;
8307
8308                 if (link_up) {
8309                         netif_carrier_on(vsi->netdev);
8310                         netif_tx_wake_all_queues(vsi->netdev);
8311                 } else {
8312                         netif_carrier_off(vsi->netdev);
8313                         netif_tx_stop_all_queues(vsi->netdev);
8314                 }
8315                 break;
8316
8317         case I40E_VSI_SRIOV:
8318         case I40E_VSI_VMDQ2:
8319         case I40E_VSI_CTRL:
8320         case I40E_VSI_IWARP:
8321         case I40E_VSI_MIRROR:
8322         default:
8323                 /* there is no notification for other VSIs */
8324                 break;
8325         }
8326 }
8327
8328 /**
8329  * i40e_veb_link_event - notify elements on the veb of a link event
8330  * @veb: veb to be notified
8331  * @link_up: link up or down
8332  **/
8333 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
8334 {
8335         struct i40e_pf *pf;
8336         int i;
8337
8338         if (!veb || !veb->pf)
8339                 return;
8340         pf = veb->pf;
8341
8342         /* depth first... */
8343         for (i = 0; i < I40E_MAX_VEB; i++)
8344                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
8345                         i40e_veb_link_event(pf->veb[i], link_up);
8346
8347         /* ... now the local VSIs */
8348         for (i = 0; i < pf->num_alloc_vsi; i++)
8349                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
8350                         i40e_vsi_link_event(pf->vsi[i], link_up);
8351 }
8352
8353 /**
8354  * i40e_link_event - Update netif_carrier status
8355  * @pf: board private structure
8356  **/
8357 static void i40e_link_event(struct i40e_pf *pf)
8358 {
8359         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8360         u8 new_link_speed, old_link_speed;
8361         i40e_status status;
8362         bool new_link, old_link;
8363
8364         /* save off old link status information */
8365         pf->hw.phy.link_info_old = pf->hw.phy.link_info;
8366
8367         /* set this to force the get_link_status call to refresh state */
8368         pf->hw.phy.get_link_info = true;
8369
8370         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
8371
8372         status = i40e_get_link_status(&pf->hw, &new_link);
8373
8374         /* On success, disable temp link polling */
8375         if (status == I40E_SUCCESS) {
8376                 if (pf->flags & I40E_FLAG_TEMP_LINK_POLLING)
8377                         pf->flags &= ~I40E_FLAG_TEMP_LINK_POLLING;
8378         } else {
8379                 /* Enable link polling temporarily until i40e_get_link_status
8380                  * returns I40E_SUCCESS
8381                  */
8382                 pf->flags |= I40E_FLAG_TEMP_LINK_POLLING;
8383                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
8384                         status);
8385                 return;
8386         }
8387
8388         old_link_speed = pf->hw.phy.link_info_old.link_speed;
8389         new_link_speed = pf->hw.phy.link_info.link_speed;
8390
8391         if (new_link == old_link &&
8392             new_link_speed == old_link_speed &&
8393             (test_bit(__I40E_VSI_DOWN, vsi->state) ||
8394              new_link == netif_carrier_ok(vsi->netdev)))
8395                 return;
8396
8397         i40e_print_link_message(vsi, new_link);
8398
8399         /* Notify the base of the switch tree connected to
8400          * the link.  Floating VEBs are not notified.
8401          */
8402         if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
8403                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
8404         else
8405                 i40e_vsi_link_event(vsi, new_link);
8406
8407         if (pf->vf)
8408                 i40e_vc_notify_link_state(pf);
8409
8410         if (pf->flags & I40E_FLAG_PTP)
8411                 i40e_ptp_set_increment(pf);
8412 }
8413
8414 /**
8415  * i40e_watchdog_subtask - periodic checks not using event driven response
8416  * @pf: board private structure
8417  **/
8418 static void i40e_watchdog_subtask(struct i40e_pf *pf)
8419 {
8420         int i;
8421
8422         /* if interface is down do nothing */
8423         if (test_bit(__I40E_DOWN, pf->state) ||
8424             test_bit(__I40E_CONFIG_BUSY, pf->state))
8425                 return;
8426
8427         /* make sure we don't do these things too often */
8428         if (time_before(jiffies, (pf->service_timer_previous +
8429                                   pf->service_timer_period)))
8430                 return;
8431         pf->service_timer_previous = jiffies;
8432
8433         if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
8434             (pf->flags & I40E_FLAG_TEMP_LINK_POLLING))
8435                 i40e_link_event(pf);
8436
8437         /* Update the stats for active netdevs so the network stack
8438          * can look at updated numbers whenever it cares to
8439          */
8440         for (i = 0; i < pf->num_alloc_vsi; i++)
8441                 if (pf->vsi[i] && pf->vsi[i]->netdev)
8442                         i40e_update_stats(pf->vsi[i]);
8443
8444         if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
8445                 /* Update the stats for the active switching components */
8446                 for (i = 0; i < I40E_MAX_VEB; i++)
8447                         if (pf->veb[i])
8448                                 i40e_update_veb_stats(pf->veb[i]);
8449         }
8450
8451         i40e_ptp_rx_hang(pf);
8452         i40e_ptp_tx_hang(pf);
8453 }
8454
8455 /**
8456  * i40e_reset_subtask - Set up for resetting the device and driver
8457  * @pf: board private structure
8458  **/
8459 static void i40e_reset_subtask(struct i40e_pf *pf)
8460 {
8461         u32 reset_flags = 0;
8462
8463         if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
8464                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
8465                 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
8466         }
8467         if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
8468                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
8469                 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
8470         }
8471         if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
8472                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
8473                 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
8474         }
8475         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
8476                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
8477                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
8478         }
8479         if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
8480                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
8481                 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
8482         }
8483
8484         /* If there's a recovery already waiting, it takes
8485          * precedence before starting a new reset sequence.
8486          */
8487         if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
8488                 i40e_prep_for_reset(pf, false);
8489                 i40e_reset(pf);
8490                 i40e_rebuild(pf, false, false);
8491         }
8492
8493         /* If we're already down or resetting, just bail */
8494         if (reset_flags &&
8495             !test_bit(__I40E_DOWN, pf->state) &&
8496             !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
8497                 i40e_do_reset(pf, reset_flags, false);
8498         }
8499 }
8500
8501 /**
8502  * i40e_handle_link_event - Handle link event
8503  * @pf: board private structure
8504  * @e: event info posted on ARQ
8505  **/
8506 static void i40e_handle_link_event(struct i40e_pf *pf,
8507                                    struct i40e_arq_event_info *e)
8508 {
8509         struct i40e_aqc_get_link_status *status =
8510                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
8511
8512         /* Do a new status request to re-enable LSE reporting
8513          * and load new status information into the hw struct
8514          * This completely ignores any state information
8515          * in the ARQ event info, instead choosing to always
8516          * issue the AQ update link status command.
8517          */
8518         i40e_link_event(pf);
8519
8520         /* Check if module meets thermal requirements */
8521         if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
8522                 dev_err(&pf->pdev->dev,
8523                         "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
8524                 dev_err(&pf->pdev->dev,
8525                         "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
8526         } else {
8527                 /* check for unqualified module, if link is down, suppress
8528                  * the message if link was forced to be down.
8529                  */
8530                 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
8531                     (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
8532                     (!(status->link_info & I40E_AQ_LINK_UP)) &&
8533                     (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) {
8534                         dev_err(&pf->pdev->dev,
8535                                 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
8536                         dev_err(&pf->pdev->dev,
8537                                 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
8538                 }
8539         }
8540 }
8541
8542 /**
8543  * i40e_clean_adminq_subtask - Clean the AdminQ rings
8544  * @pf: board private structure
8545  **/
8546 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
8547 {
8548         struct i40e_arq_event_info event;
8549         struct i40e_hw *hw = &pf->hw;
8550         u16 pending, i = 0;
8551         i40e_status ret;
8552         u16 opcode;
8553         u32 oldval;
8554         u32 val;
8555
8556         /* Do not run clean AQ when PF reset fails */
8557         if (test_bit(__I40E_RESET_FAILED, pf->state))
8558                 return;
8559
8560         /* check for error indications */
8561         val = rd32(&pf->hw, pf->hw.aq.arq.len);
8562         oldval = val;
8563         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
8564                 if (hw->debug_mask & I40E_DEBUG_AQ)
8565                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
8566                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
8567         }
8568         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
8569                 if (hw->debug_mask & I40E_DEBUG_AQ)
8570                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
8571                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
8572                 pf->arq_overflows++;
8573         }
8574         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
8575                 if (hw->debug_mask & I40E_DEBUG_AQ)
8576                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
8577                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
8578         }
8579         if (oldval != val)
8580                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
8581
8582         val = rd32(&pf->hw, pf->hw.aq.asq.len);
8583         oldval = val;
8584         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
8585                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
8586                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
8587                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
8588         }
8589         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
8590                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
8591                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
8592                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
8593         }
8594         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
8595                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
8596                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
8597                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
8598         }
8599         if (oldval != val)
8600                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
8601
8602         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
8603         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
8604         if (!event.msg_buf)
8605                 return;
8606
8607         do {
8608                 ret = i40e_clean_arq_element(hw, &event, &pending);
8609                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
8610                         break;
8611                 else if (ret) {
8612                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
8613                         break;
8614                 }
8615
8616                 opcode = le16_to_cpu(event.desc.opcode);
8617                 switch (opcode) {
8618
8619                 case i40e_aqc_opc_get_link_status:
8620                         i40e_handle_link_event(pf, &event);
8621                         break;
8622                 case i40e_aqc_opc_send_msg_to_pf:
8623                         ret = i40e_vc_process_vf_msg(pf,
8624                                         le16_to_cpu(event.desc.retval),
8625                                         le32_to_cpu(event.desc.cookie_high),
8626                                         le32_to_cpu(event.desc.cookie_low),
8627                                         event.msg_buf,
8628                                         event.msg_len);
8629                         break;
8630                 case i40e_aqc_opc_lldp_update_mib:
8631                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
8632 #ifdef CONFIG_I40E_DCB
8633                         rtnl_lock();
8634                         ret = i40e_handle_lldp_event(pf, &event);
8635                         rtnl_unlock();
8636 #endif /* CONFIG_I40E_DCB */
8637                         break;
8638                 case i40e_aqc_opc_event_lan_overflow:
8639                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
8640                         i40e_handle_lan_overflow_event(pf, &event);
8641                         break;
8642                 case i40e_aqc_opc_send_msg_to_peer:
8643                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
8644                         break;
8645                 case i40e_aqc_opc_nvm_erase:
8646                 case i40e_aqc_opc_nvm_update:
8647                 case i40e_aqc_opc_oem_post_update:
8648                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
8649                                    "ARQ NVM operation 0x%04x completed\n",
8650                                    opcode);
8651                         break;
8652                 default:
8653                         dev_info(&pf->pdev->dev,
8654                                  "ARQ: Unknown event 0x%04x ignored\n",
8655                                  opcode);
8656                         break;
8657                 }
8658         } while (i++ < pf->adminq_work_limit);
8659
8660         if (i < pf->adminq_work_limit)
8661                 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
8662
8663         /* re-enable Admin queue interrupt cause */
8664         val = rd32(hw, I40E_PFINT_ICR0_ENA);
8665         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
8666         wr32(hw, I40E_PFINT_ICR0_ENA, val);
8667         i40e_flush(hw);
8668
8669         kfree(event.msg_buf);
8670 }
8671
8672 /**
8673  * i40e_verify_eeprom - make sure eeprom is good to use
8674  * @pf: board private structure
8675  **/
8676 static void i40e_verify_eeprom(struct i40e_pf *pf)
8677 {
8678         int err;
8679
8680         err = i40e_diag_eeprom_test(&pf->hw);
8681         if (err) {
8682                 /* retry in case of garbage read */
8683                 err = i40e_diag_eeprom_test(&pf->hw);
8684                 if (err) {
8685                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
8686                                  err);
8687                         set_bit(__I40E_BAD_EEPROM, pf->state);
8688                 }
8689         }
8690
8691         if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
8692                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
8693                 clear_bit(__I40E_BAD_EEPROM, pf->state);
8694         }
8695 }
8696
8697 /**
8698  * i40e_enable_pf_switch_lb
8699  * @pf: pointer to the PF structure
8700  *
8701  * enable switch loop back or die - no point in a return value
8702  **/
8703 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
8704 {
8705         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8706         struct i40e_vsi_context ctxt;
8707         int ret;
8708
8709         ctxt.seid = pf->main_vsi_seid;
8710         ctxt.pf_num = pf->hw.pf_id;
8711         ctxt.vf_num = 0;
8712         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
8713         if (ret) {
8714                 dev_info(&pf->pdev->dev,
8715                          "couldn't get PF vsi config, err %s aq_err %s\n",
8716                          i40e_stat_str(&pf->hw, ret),
8717                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
8718                 return;
8719         }
8720         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
8721         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
8722         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
8723
8724         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
8725         if (ret) {
8726                 dev_info(&pf->pdev->dev,
8727                          "update vsi switch failed, err %s aq_err %s\n",
8728                          i40e_stat_str(&pf->hw, ret),
8729                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
8730         }
8731 }
8732
8733 /**
8734  * i40e_disable_pf_switch_lb
8735  * @pf: pointer to the PF structure
8736  *
8737  * disable switch loop back or die - no point in a return value
8738  **/
8739 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
8740 {
8741         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8742         struct i40e_vsi_context ctxt;
8743         int ret;
8744
8745         ctxt.seid = pf->main_vsi_seid;
8746         ctxt.pf_num = pf->hw.pf_id;
8747         ctxt.vf_num = 0;
8748         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
8749         if (ret) {
8750                 dev_info(&pf->pdev->dev,
8751                          "couldn't get PF vsi config, err %s aq_err %s\n",
8752                          i40e_stat_str(&pf->hw, ret),
8753                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
8754                 return;
8755         }
8756         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
8757         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
8758         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
8759
8760         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
8761         if (ret) {
8762                 dev_info(&pf->pdev->dev,
8763                          "update vsi switch failed, err %s aq_err %s\n",
8764                          i40e_stat_str(&pf->hw, ret),
8765                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
8766         }
8767 }
8768
8769 /**
8770  * i40e_config_bridge_mode - Configure the HW bridge mode
8771  * @veb: pointer to the bridge instance
8772  *
8773  * Configure the loop back mode for the LAN VSI that is downlink to the
8774  * specified HW bridge instance. It is expected this function is called
8775  * when a new HW bridge is instantiated.
8776  **/
8777 static void i40e_config_bridge_mode(struct i40e_veb *veb)
8778 {
8779         struct i40e_pf *pf = veb->pf;
8780
8781         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
8782                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
8783                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
8784         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
8785                 i40e_disable_pf_switch_lb(pf);
8786         else
8787                 i40e_enable_pf_switch_lb(pf);
8788 }
8789
8790 /**
8791  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
8792  * @veb: pointer to the VEB instance
8793  *
8794  * This is a recursive function that first builds the attached VSIs then
8795  * recurses in to build the next layer of VEB.  We track the connections
8796  * through our own index numbers because the seid's from the HW could
8797  * change across the reset.
8798  **/
8799 static int i40e_reconstitute_veb(struct i40e_veb *veb)
8800 {
8801         struct i40e_vsi *ctl_vsi = NULL;
8802         struct i40e_pf *pf = veb->pf;
8803         int v, veb_idx;
8804         int ret;
8805
8806         /* build VSI that owns this VEB, temporarily attached to base VEB */
8807         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
8808                 if (pf->vsi[v] &&
8809                     pf->vsi[v]->veb_idx == veb->idx &&
8810                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
8811                         ctl_vsi = pf->vsi[v];
8812                         break;
8813                 }
8814         }
8815         if (!ctl_vsi) {
8816                 dev_info(&pf->pdev->dev,
8817                          "missing owner VSI for veb_idx %d\n", veb->idx);
8818                 ret = -ENOENT;
8819                 goto end_reconstitute;
8820         }
8821         if (ctl_vsi != pf->vsi[pf->lan_vsi])
8822                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
8823         ret = i40e_add_vsi(ctl_vsi);
8824         if (ret) {
8825                 dev_info(&pf->pdev->dev,
8826                          "rebuild of veb_idx %d owner VSI failed: %d\n",
8827                          veb->idx, ret);
8828                 goto end_reconstitute;
8829         }
8830         i40e_vsi_reset_stats(ctl_vsi);
8831
8832         /* create the VEB in the switch and move the VSI onto the VEB */
8833         ret = i40e_add_veb(veb, ctl_vsi);
8834         if (ret)
8835                 goto end_reconstitute;
8836
8837         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
8838                 veb->bridge_mode = BRIDGE_MODE_VEB;
8839         else
8840                 veb->bridge_mode = BRIDGE_MODE_VEPA;
8841         i40e_config_bridge_mode(veb);
8842
8843         /* create the remaining VSIs attached to this VEB */
8844         for (v = 0; v < pf->num_alloc_vsi; v++) {
8845                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
8846                         continue;
8847
8848                 if (pf->vsi[v]->veb_idx == veb->idx) {
8849                         struct i40e_vsi *vsi = pf->vsi[v];
8850
8851                         vsi->uplink_seid = veb->seid;
8852                         ret = i40e_add_vsi(vsi);
8853                         if (ret) {
8854                                 dev_info(&pf->pdev->dev,
8855                                          "rebuild of vsi_idx %d failed: %d\n",
8856                                          v, ret);
8857                                 goto end_reconstitute;
8858                         }
8859                         i40e_vsi_reset_stats(vsi);
8860                 }
8861         }
8862
8863         /* create any VEBs attached to this VEB - RECURSION */
8864         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
8865                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
8866                         pf->veb[veb_idx]->uplink_seid = veb->seid;
8867                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
8868                         if (ret)
8869                                 break;
8870                 }
8871         }
8872
8873 end_reconstitute:
8874         return ret;
8875 }
8876
8877 /**
8878  * i40e_get_capabilities - get info about the HW
8879  * @pf: the PF struct
8880  **/
8881 static int i40e_get_capabilities(struct i40e_pf *pf,
8882                                  enum i40e_admin_queue_opc list_type)
8883 {
8884         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
8885         u16 data_size;
8886         int buf_len;
8887         int err;
8888
8889         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
8890         do {
8891                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
8892                 if (!cap_buf)
8893                         return -ENOMEM;
8894
8895                 /* this loads the data into the hw struct for us */
8896                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
8897                                                     &data_size, list_type,
8898                                                     NULL);
8899                 /* data loaded, buffer no longer needed */
8900                 kfree(cap_buf);
8901
8902                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
8903                         /* retry with a larger buffer */
8904                         buf_len = data_size;
8905                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
8906                         dev_info(&pf->pdev->dev,
8907                                  "capability discovery failed, err %s aq_err %s\n",
8908                                  i40e_stat_str(&pf->hw, err),
8909                                  i40e_aq_str(&pf->hw,
8910                                              pf->hw.aq.asq_last_status));
8911                         return -ENODEV;
8912                 }
8913         } while (err);
8914
8915         if (pf->hw.debug_mask & I40E_DEBUG_USER) {
8916                 if (list_type == i40e_aqc_opc_list_func_capabilities) {
8917                         dev_info(&pf->pdev->dev,
8918                                  "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
8919                                  pf->hw.pf_id, pf->hw.func_caps.num_vfs,
8920                                  pf->hw.func_caps.num_msix_vectors,
8921                                  pf->hw.func_caps.num_msix_vectors_vf,
8922                                  pf->hw.func_caps.fd_filters_guaranteed,
8923                                  pf->hw.func_caps.fd_filters_best_effort,
8924                                  pf->hw.func_caps.num_tx_qp,
8925                                  pf->hw.func_caps.num_vsis);
8926                 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
8927                         dev_info(&pf->pdev->dev,
8928                                  "switch_mode=0x%04x, function_valid=0x%08x\n",
8929                                  pf->hw.dev_caps.switch_mode,
8930                                  pf->hw.dev_caps.valid_functions);
8931                         dev_info(&pf->pdev->dev,
8932                                  "SR-IOV=%d, num_vfs for all function=%u\n",
8933                                  pf->hw.dev_caps.sr_iov_1_1,
8934                                  pf->hw.dev_caps.num_vfs);
8935                         dev_info(&pf->pdev->dev,
8936                                  "num_vsis=%u, num_rx:%u, num_tx=%u\n",
8937                                  pf->hw.dev_caps.num_vsis,
8938                                  pf->hw.dev_caps.num_rx_qp,
8939                                  pf->hw.dev_caps.num_tx_qp);
8940                 }
8941         }
8942         if (list_type == i40e_aqc_opc_list_func_capabilities) {
8943 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
8944                        + pf->hw.func_caps.num_vfs)
8945                 if (pf->hw.revision_id == 0 &&
8946                     pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
8947                         dev_info(&pf->pdev->dev,
8948                                  "got num_vsis %d, setting num_vsis to %d\n",
8949                                  pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
8950                         pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
8951                 }
8952         }
8953         return 0;
8954 }
8955
8956 static int i40e_vsi_clear(struct i40e_vsi *vsi);
8957
8958 /**
8959  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
8960  * @pf: board private structure
8961  **/
8962 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
8963 {
8964         struct i40e_vsi *vsi;
8965
8966         /* quick workaround for an NVM issue that leaves a critical register
8967          * uninitialized
8968          */
8969         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
8970                 static const u32 hkey[] = {
8971                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
8972                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
8973                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
8974                         0x95b3a76d};
8975                 int i;
8976
8977                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
8978                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
8979         }
8980
8981         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
8982                 return;
8983
8984         /* find existing VSI and see if it needs configuring */
8985         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
8986
8987         /* create a new VSI if none exists */
8988         if (!vsi) {
8989                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
8990                                      pf->vsi[pf->lan_vsi]->seid, 0);
8991                 if (!vsi) {
8992                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
8993                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8994                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
8995                         return;
8996                 }
8997         }
8998
8999         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
9000 }
9001
9002 /**
9003  * i40e_fdir_teardown - release the Flow Director resources
9004  * @pf: board private structure
9005  **/
9006 static void i40e_fdir_teardown(struct i40e_pf *pf)
9007 {
9008         struct i40e_vsi *vsi;
9009
9010         i40e_fdir_filter_exit(pf);
9011         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
9012         if (vsi)
9013                 i40e_vsi_release(vsi);
9014 }
9015
9016 /**
9017  * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
9018  * @vsi: PF main vsi
9019  * @seid: seid of main or channel VSIs
9020  *
9021  * Rebuilds cloud filters associated with main VSI and channel VSIs if they
9022  * existed before reset
9023  **/
9024 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
9025 {
9026         struct i40e_cloud_filter *cfilter;
9027         struct i40e_pf *pf = vsi->back;
9028         struct hlist_node *node;
9029         i40e_status ret;
9030
9031         /* Add cloud filters back if they exist */
9032         hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
9033                                   cloud_node) {
9034                 if (cfilter->seid != seid)
9035                         continue;
9036
9037                 if (cfilter->dst_port)
9038                         ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
9039                                                                 true);
9040                 else
9041                         ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
9042
9043                 if (ret) {
9044                         dev_dbg(&pf->pdev->dev,
9045                                 "Failed to rebuild cloud filter, err %s aq_err %s\n",
9046                                 i40e_stat_str(&pf->hw, ret),
9047                                 i40e_aq_str(&pf->hw,
9048                                             pf->hw.aq.asq_last_status));
9049                         return ret;
9050                 }
9051         }
9052         return 0;
9053 }
9054
9055 /**
9056  * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
9057  * @vsi: PF main vsi
9058  *
9059  * Rebuilds channel VSIs if they existed before reset
9060  **/
9061 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
9062 {
9063         struct i40e_channel *ch, *ch_tmp;
9064         i40e_status ret;
9065
9066         if (list_empty(&vsi->ch_list))
9067                 return 0;
9068
9069         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
9070                 if (!ch->initialized)
9071                         break;
9072                 /* Proceed with creation of channel (VMDq2) VSI */
9073                 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
9074                 if (ret) {
9075                         dev_info(&vsi->back->pdev->dev,
9076                                  "failed to rebuild channels using uplink_seid %u\n",
9077                                  vsi->uplink_seid);
9078                         return ret;
9079                 }
9080                 if (ch->max_tx_rate) {
9081                         u64 credits = ch->max_tx_rate;
9082
9083                         if (i40e_set_bw_limit(vsi, ch->seid,
9084                                               ch->max_tx_rate))
9085                                 return -EINVAL;
9086
9087                         do_div(credits, I40E_BW_CREDIT_DIVISOR);
9088                         dev_dbg(&vsi->back->pdev->dev,
9089                                 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
9090                                 ch->max_tx_rate,
9091                                 credits,
9092                                 ch->seid);
9093                 }
9094                 ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
9095                 if (ret) {
9096                         dev_dbg(&vsi->back->pdev->dev,
9097                                 "Failed to rebuild cloud filters for channel VSI %u\n",
9098                                 ch->seid);
9099                         return ret;
9100                 }
9101         }
9102         return 0;
9103 }
9104
9105 /**
9106  * i40e_prep_for_reset - prep for the core to reset
9107  * @pf: board private structure
9108  * @lock_acquired: indicates whether or not the lock has been acquired
9109  * before this function was called.
9110  *
9111  * Close up the VFs and other things in prep for PF Reset.
9112   **/
9113 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired)
9114 {
9115         struct i40e_hw *hw = &pf->hw;
9116         i40e_status ret = 0;
9117         u32 v;
9118
9119         clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
9120         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
9121                 return;
9122         if (i40e_check_asq_alive(&pf->hw))
9123                 i40e_vc_notify_reset(pf);
9124
9125         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
9126
9127         /* quiesce the VSIs and their queues that are not already DOWN */
9128         /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */
9129         if (!lock_acquired)
9130                 rtnl_lock();
9131         i40e_pf_quiesce_all_vsi(pf);
9132         if (!lock_acquired)
9133                 rtnl_unlock();
9134
9135         for (v = 0; v < pf->num_alloc_vsi; v++) {
9136                 if (pf->vsi[v])
9137                         pf->vsi[v]->seid = 0;
9138         }
9139
9140         i40e_shutdown_adminq(&pf->hw);
9141
9142         /* call shutdown HMC */
9143         if (hw->hmc.hmc_obj) {
9144                 ret = i40e_shutdown_lan_hmc(hw);
9145                 if (ret)
9146                         dev_warn(&pf->pdev->dev,
9147                                  "shutdown_lan_hmc failed: %d\n", ret);
9148         }
9149 }
9150
9151 /**
9152  * i40e_send_version - update firmware with driver version
9153  * @pf: PF struct
9154  */
9155 static void i40e_send_version(struct i40e_pf *pf)
9156 {
9157         struct i40e_driver_version dv;
9158
9159         dv.major_version = DRV_VERSION_MAJOR;
9160         dv.minor_version = DRV_VERSION_MINOR;
9161         dv.build_version = DRV_VERSION_BUILD;
9162         dv.subbuild_version = 0;
9163         strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
9164         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
9165 }
9166
9167 /**
9168  * i40e_get_oem_version - get OEM specific version information
9169  * @hw: pointer to the hardware structure
9170  **/
9171 static void i40e_get_oem_version(struct i40e_hw *hw)
9172 {
9173         u16 block_offset = 0xffff;
9174         u16 block_length = 0;
9175         u16 capabilities = 0;
9176         u16 gen_snap = 0;
9177         u16 release = 0;
9178
9179 #define I40E_SR_NVM_OEM_VERSION_PTR             0x1B
9180 #define I40E_NVM_OEM_LENGTH_OFFSET              0x00
9181 #define I40E_NVM_OEM_CAPABILITIES_OFFSET        0x01
9182 #define I40E_NVM_OEM_GEN_OFFSET                 0x02
9183 #define I40E_NVM_OEM_RELEASE_OFFSET             0x03
9184 #define I40E_NVM_OEM_CAPABILITIES_MASK          0x000F
9185 #define I40E_NVM_OEM_LENGTH                     3
9186
9187         /* Check if pointer to OEM version block is valid. */
9188         i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
9189         if (block_offset == 0xffff)
9190                 return;
9191
9192         /* Check if OEM version block has correct length. */
9193         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
9194                            &block_length);
9195         if (block_length < I40E_NVM_OEM_LENGTH)
9196                 return;
9197
9198         /* Check if OEM version format is as expected. */
9199         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
9200                            &capabilities);
9201         if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
9202                 return;
9203
9204         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
9205                            &gen_snap);
9206         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
9207                            &release);
9208         hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
9209         hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
9210 }
9211
9212 /**
9213  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
9214  * @pf: board private structure
9215  **/
9216 static int i40e_reset(struct i40e_pf *pf)
9217 {
9218         struct i40e_hw *hw = &pf->hw;
9219         i40e_status ret;
9220
9221         ret = i40e_pf_reset(hw);
9222         if (ret) {
9223                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
9224                 set_bit(__I40E_RESET_FAILED, pf->state);
9225                 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
9226         } else {
9227                 pf->pfr_count++;
9228         }
9229         return ret;
9230 }
9231
9232 /**
9233  * i40e_rebuild - rebuild using a saved config
9234  * @pf: board private structure
9235  * @reinit: if the Main VSI needs to re-initialized.
9236  * @lock_acquired: indicates whether or not the lock has been acquired
9237  * before this function was called.
9238  **/
9239 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
9240 {
9241         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9242         struct i40e_hw *hw = &pf->hw;
9243         u8 set_fc_aq_fail = 0;
9244         i40e_status ret;
9245         u32 val;
9246         int v;
9247
9248         if (test_bit(__I40E_DOWN, pf->state))
9249                 goto clear_recovery;
9250         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
9251
9252         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
9253         ret = i40e_init_adminq(&pf->hw);
9254         if (ret) {
9255                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
9256                          i40e_stat_str(&pf->hw, ret),
9257                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9258                 goto clear_recovery;
9259         }
9260         i40e_get_oem_version(&pf->hw);
9261
9262         /* re-verify the eeprom if we just had an EMP reset */
9263         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
9264                 i40e_verify_eeprom(pf);
9265
9266         i40e_clear_pxe_mode(hw);
9267         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
9268         if (ret)
9269                 goto end_core_reset;
9270
9271         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
9272                                 hw->func_caps.num_rx_qp, 0, 0);
9273         if (ret) {
9274                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
9275                 goto end_core_reset;
9276         }
9277         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
9278         if (ret) {
9279                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
9280                 goto end_core_reset;
9281         }
9282
9283 #ifdef CONFIG_I40E_DCB
9284         ret = i40e_init_pf_dcb(pf);
9285         if (ret) {
9286                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret);
9287                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
9288                 /* Continue without DCB enabled */
9289         }
9290 #endif /* CONFIG_I40E_DCB */
9291         /* do basic switch setup */
9292         if (!lock_acquired)
9293                 rtnl_lock();
9294         ret = i40e_setup_pf_switch(pf, reinit);
9295         if (ret)
9296                 goto end_unlock;
9297
9298         /* The driver only wants link up/down and module qualification
9299          * reports from firmware.  Note the negative logic.
9300          */
9301         ret = i40e_aq_set_phy_int_mask(&pf->hw,
9302                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
9303                                          I40E_AQ_EVENT_MEDIA_NA |
9304                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
9305         if (ret)
9306                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
9307                          i40e_stat_str(&pf->hw, ret),
9308                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9309
9310         /* make sure our flow control settings are restored */
9311         ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true);
9312         if (ret)
9313                 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n",
9314                         i40e_stat_str(&pf->hw, ret),
9315                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9316
9317         /* Rebuild the VSIs and VEBs that existed before reset.
9318          * They are still in our local switch element arrays, so only
9319          * need to rebuild the switch model in the HW.
9320          *
9321          * If there were VEBs but the reconstitution failed, we'll try
9322          * try to recover minimal use by getting the basic PF VSI working.
9323          */
9324         if (vsi->uplink_seid != pf->mac_seid) {
9325                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
9326                 /* find the one VEB connected to the MAC, and find orphans */
9327                 for (v = 0; v < I40E_MAX_VEB; v++) {
9328                         if (!pf->veb[v])
9329                                 continue;
9330
9331                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
9332                             pf->veb[v]->uplink_seid == 0) {
9333                                 ret = i40e_reconstitute_veb(pf->veb[v]);
9334
9335                                 if (!ret)
9336                                         continue;
9337
9338                                 /* If Main VEB failed, we're in deep doodoo,
9339                                  * so give up rebuilding the switch and set up
9340                                  * for minimal rebuild of PF VSI.
9341                                  * If orphan failed, we'll report the error
9342                                  * but try to keep going.
9343                                  */
9344                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
9345                                         dev_info(&pf->pdev->dev,
9346                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
9347                                                  ret);
9348                                         vsi->uplink_seid = pf->mac_seid;
9349                                         break;
9350                                 } else if (pf->veb[v]->uplink_seid == 0) {
9351                                         dev_info(&pf->pdev->dev,
9352                                                  "rebuild of orphan VEB failed: %d\n",
9353                                                  ret);
9354                                 }
9355                         }
9356                 }
9357         }
9358
9359         if (vsi->uplink_seid == pf->mac_seid) {
9360                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
9361                 /* no VEB, so rebuild only the Main VSI */
9362                 ret = i40e_add_vsi(vsi);
9363                 if (ret) {
9364                         dev_info(&pf->pdev->dev,
9365                                  "rebuild of Main VSI failed: %d\n", ret);
9366                         goto end_unlock;
9367                 }
9368         }
9369
9370         if (vsi->mqprio_qopt.max_rate[0]) {
9371                 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
9372                 u64 credits = 0;
9373
9374                 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
9375                 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
9376                 if (ret)
9377                         goto end_unlock;
9378
9379                 credits = max_tx_rate;
9380                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
9381                 dev_dbg(&vsi->back->pdev->dev,
9382                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
9383                         max_tx_rate,
9384                         credits,
9385                         vsi->seid);
9386         }
9387
9388         ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
9389         if (ret)
9390                 goto end_unlock;
9391
9392         /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
9393          * for this main VSI if they exist
9394          */
9395         ret = i40e_rebuild_channels(vsi);
9396         if (ret)
9397                 goto end_unlock;
9398
9399         /* Reconfigure hardware for allowing smaller MSS in the case
9400          * of TSO, so that we avoid the MDD being fired and causing
9401          * a reset in the case of small MSS+TSO.
9402          */
9403 #define I40E_REG_MSS          0x000E64DC
9404 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
9405 #define I40E_64BYTE_MSS       0x400000
9406         val = rd32(hw, I40E_REG_MSS);
9407         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
9408                 val &= ~I40E_REG_MSS_MIN_MASK;
9409                 val |= I40E_64BYTE_MSS;
9410                 wr32(hw, I40E_REG_MSS, val);
9411         }
9412
9413         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
9414                 msleep(75);
9415                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
9416                 if (ret)
9417                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
9418                                  i40e_stat_str(&pf->hw, ret),
9419                                  i40e_aq_str(&pf->hw,
9420                                              pf->hw.aq.asq_last_status));
9421         }
9422         /* reinit the misc interrupt */
9423         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
9424                 ret = i40e_setup_misc_vector(pf);
9425
9426         /* Add a filter to drop all Flow control frames from any VSI from being
9427          * transmitted. By doing so we stop a malicious VF from sending out
9428          * PAUSE or PFC frames and potentially controlling traffic for other
9429          * PF/VF VSIs.
9430          * The FW can still send Flow control frames if enabled.
9431          */
9432         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
9433                                                        pf->main_vsi_seid);
9434
9435         /* restart the VSIs that were rebuilt and running before the reset */
9436         i40e_pf_unquiesce_all_vsi(pf);
9437
9438         /* Release the RTNL lock before we start resetting VFs */
9439         if (!lock_acquired)
9440                 rtnl_unlock();
9441
9442         /* Restore promiscuous settings */
9443         ret = i40e_set_promiscuous(pf, pf->cur_promisc);
9444         if (ret)
9445                 dev_warn(&pf->pdev->dev,
9446                          "Failed to restore promiscuous setting: %s, err %s aq_err %s\n",
9447                          pf->cur_promisc ? "on" : "off",
9448                          i40e_stat_str(&pf->hw, ret),
9449                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9450
9451         i40e_reset_all_vfs(pf, true);
9452
9453         /* tell the firmware that we're starting */
9454         i40e_send_version(pf);
9455
9456         /* We've already released the lock, so don't do it again */
9457         goto end_core_reset;
9458
9459 end_unlock:
9460         if (!lock_acquired)
9461                 rtnl_unlock();
9462 end_core_reset:
9463         clear_bit(__I40E_RESET_FAILED, pf->state);
9464 clear_recovery:
9465         clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
9466 }
9467
9468 /**
9469  * i40e_reset_and_rebuild - reset and rebuild using a saved config
9470  * @pf: board private structure
9471  * @reinit: if the Main VSI needs to re-initialized.
9472  * @lock_acquired: indicates whether or not the lock has been acquired
9473  * before this function was called.
9474  **/
9475 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
9476                                    bool lock_acquired)
9477 {
9478         int ret;
9479         /* Now we wait for GRST to settle out.
9480          * We don't have to delete the VEBs or VSIs from the hw switch
9481          * because the reset will make them disappear.
9482          */
9483         ret = i40e_reset(pf);
9484         if (!ret)
9485                 i40e_rebuild(pf, reinit, lock_acquired);
9486 }
9487
9488 /**
9489  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
9490  * @pf: board private structure
9491  *
9492  * Close up the VFs and other things in prep for a Core Reset,
9493  * then get ready to rebuild the world.
9494  * @lock_acquired: indicates whether or not the lock has been acquired
9495  * before this function was called.
9496  **/
9497 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
9498 {
9499         i40e_prep_for_reset(pf, lock_acquired);
9500         i40e_reset_and_rebuild(pf, false, lock_acquired);
9501 }
9502
9503 /**
9504  * i40e_handle_mdd_event
9505  * @pf: pointer to the PF structure
9506  *
9507  * Called from the MDD irq handler to identify possibly malicious vfs
9508  **/
9509 static void i40e_handle_mdd_event(struct i40e_pf *pf)
9510 {
9511         struct i40e_hw *hw = &pf->hw;
9512         bool mdd_detected = false;
9513         bool pf_mdd_detected = false;
9514         struct i40e_vf *vf;
9515         u32 reg;
9516         int i;
9517
9518         if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
9519                 return;
9520
9521         /* find what triggered the MDD event */
9522         reg = rd32(hw, I40E_GL_MDET_TX);
9523         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
9524                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
9525                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
9526                 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
9527                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
9528                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
9529                                 I40E_GL_MDET_TX_EVENT_SHIFT;
9530                 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
9531                                 I40E_GL_MDET_TX_QUEUE_SHIFT) -
9532                                 pf->hw.func_caps.base_queue;
9533                 if (netif_msg_tx_err(pf))
9534                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
9535                                  event, queue, pf_num, vf_num);
9536                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
9537                 mdd_detected = true;
9538         }
9539         reg = rd32(hw, I40E_GL_MDET_RX);
9540         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
9541                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
9542                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
9543                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
9544                                 I40E_GL_MDET_RX_EVENT_SHIFT;
9545                 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
9546                                 I40E_GL_MDET_RX_QUEUE_SHIFT) -
9547                                 pf->hw.func_caps.base_queue;
9548                 if (netif_msg_rx_err(pf))
9549                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
9550                                  event, queue, func);
9551                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
9552                 mdd_detected = true;
9553         }
9554
9555         if (mdd_detected) {
9556                 reg = rd32(hw, I40E_PF_MDET_TX);
9557                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
9558                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
9559                         dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n");
9560                         pf_mdd_detected = true;
9561                 }
9562                 reg = rd32(hw, I40E_PF_MDET_RX);
9563                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
9564                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
9565                         dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n");
9566                         pf_mdd_detected = true;
9567                 }
9568                 /* Queue belongs to the PF, initiate a reset */
9569                 if (pf_mdd_detected) {
9570                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9571                         i40e_service_event_schedule(pf);
9572                 }
9573         }
9574
9575         /* see if one of the VFs needs its hand slapped */
9576         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
9577                 vf = &(pf->vf[i]);
9578                 reg = rd32(hw, I40E_VP_MDET_TX(i));
9579                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
9580                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
9581                         vf->num_mdd_events++;
9582                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
9583                                  i);
9584                 }
9585
9586                 reg = rd32(hw, I40E_VP_MDET_RX(i));
9587                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
9588                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
9589                         vf->num_mdd_events++;
9590                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
9591                                  i);
9592                 }
9593
9594                 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
9595                         dev_info(&pf->pdev->dev,
9596                                  "Too many MDD events on VF %d, disabled\n", i);
9597                         dev_info(&pf->pdev->dev,
9598                                  "Use PF Control I/F to re-enable the VF\n");
9599                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
9600                 }
9601         }
9602
9603         /* re-enable mdd interrupt cause */
9604         clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
9605         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
9606         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
9607         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
9608         i40e_flush(hw);
9609 }
9610
9611 static const char *i40e_tunnel_name(struct i40e_udp_port_config *port)
9612 {
9613         switch (port->type) {
9614         case UDP_TUNNEL_TYPE_VXLAN:
9615                 return "vxlan";
9616         case UDP_TUNNEL_TYPE_GENEVE:
9617                 return "geneve";
9618         default:
9619                 return "unknown";
9620         }
9621 }
9622
9623 /**
9624  * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters
9625  * @pf: board private structure
9626  **/
9627 static void i40e_sync_udp_filters(struct i40e_pf *pf)
9628 {
9629         int i;
9630
9631         /* loop through and set pending bit for all active UDP filters */
9632         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
9633                 if (pf->udp_ports[i].port)
9634                         pf->pending_udp_bitmap |= BIT_ULL(i);
9635         }
9636
9637         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
9638 }
9639
9640 /**
9641  * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW
9642  * @pf: board private structure
9643  **/
9644 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
9645 {
9646         struct i40e_hw *hw = &pf->hw;
9647         i40e_status ret;
9648         u16 port;
9649         int i;
9650
9651         if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC))
9652                 return;
9653
9654         pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC;
9655
9656         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
9657                 if (pf->pending_udp_bitmap & BIT_ULL(i)) {
9658                         pf->pending_udp_bitmap &= ~BIT_ULL(i);
9659                         port = pf->udp_ports[i].port;
9660                         if (port)
9661                                 ret = i40e_aq_add_udp_tunnel(hw, port,
9662                                                         pf->udp_ports[i].type,
9663                                                         NULL, NULL);
9664                         else
9665                                 ret = i40e_aq_del_udp_tunnel(hw, i, NULL);
9666
9667                         if (ret) {
9668                                 dev_info(&pf->pdev->dev,
9669                                          "%s %s port %d, index %d failed, err %s aq_err %s\n",
9670                                          i40e_tunnel_name(&pf->udp_ports[i]),
9671                                          port ? "add" : "delete",
9672                                          port, i,
9673                                          i40e_stat_str(&pf->hw, ret),
9674                                          i40e_aq_str(&pf->hw,
9675                                                      pf->hw.aq.asq_last_status));
9676                                 pf->udp_ports[i].port = 0;
9677                         }
9678                 }
9679         }
9680 }
9681
9682 /**
9683  * i40e_service_task - Run the driver's async subtasks
9684  * @work: pointer to work_struct containing our data
9685  **/
9686 static void i40e_service_task(struct work_struct *work)
9687 {
9688         struct i40e_pf *pf = container_of(work,
9689                                           struct i40e_pf,
9690                                           service_task);
9691         unsigned long start_time = jiffies;
9692
9693         /* don't bother with service tasks if a reset is in progress */
9694         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
9695                 return;
9696
9697         if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
9698                 return;
9699
9700         i40e_detect_recover_hung(pf);
9701         i40e_sync_filters_subtask(pf);
9702         i40e_reset_subtask(pf);
9703         i40e_handle_mdd_event(pf);
9704         i40e_vc_process_vflr_event(pf);
9705         i40e_watchdog_subtask(pf);
9706         i40e_fdir_reinit_subtask(pf);
9707         if (pf->flags & I40E_FLAG_CLIENT_RESET) {
9708                 /* Client subtask will reopen next time through. */
9709                 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true);
9710                 pf->flags &= ~I40E_FLAG_CLIENT_RESET;
9711         } else {
9712                 i40e_client_subtask(pf);
9713                 if (pf->flags & I40E_FLAG_CLIENT_L2_CHANGE) {
9714                         i40e_notify_client_of_l2_param_changes(
9715                                                         pf->vsi[pf->lan_vsi]);
9716                         pf->flags &= ~I40E_FLAG_CLIENT_L2_CHANGE;
9717                 }
9718         }
9719         i40e_sync_filters_subtask(pf);
9720         i40e_sync_udp_filters_subtask(pf);
9721         i40e_clean_adminq_subtask(pf);
9722
9723         /* flush memory to make sure state is correct before next watchdog */
9724         smp_mb__before_atomic();
9725         clear_bit(__I40E_SERVICE_SCHED, pf->state);
9726
9727         /* If the tasks have taken longer than one timer cycle or there
9728          * is more work to be done, reschedule the service task now
9729          * rather than wait for the timer to tick again.
9730          */
9731         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
9732             test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)             ||
9733             test_bit(__I40E_MDD_EVENT_PENDING, pf->state)                ||
9734             test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
9735                 i40e_service_event_schedule(pf);
9736 }
9737
9738 /**
9739  * i40e_service_timer - timer callback
9740  * @data: pointer to PF struct
9741  **/
9742 static void i40e_service_timer(struct timer_list *t)
9743 {
9744         struct i40e_pf *pf = from_timer(pf, t, service_timer);
9745
9746         mod_timer(&pf->service_timer,
9747                   round_jiffies(jiffies + pf->service_timer_period));
9748         i40e_service_event_schedule(pf);
9749 }
9750
9751 /**
9752  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
9753  * @vsi: the VSI being configured
9754  **/
9755 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
9756 {
9757         struct i40e_pf *pf = vsi->back;
9758
9759         switch (vsi->type) {
9760         case I40E_VSI_MAIN:
9761                 vsi->alloc_queue_pairs = pf->num_lan_qps;
9762                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
9763                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
9764                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
9765                         vsi->num_q_vectors = pf->num_lan_msix;
9766                 else
9767                         vsi->num_q_vectors = 1;
9768
9769                 break;
9770
9771         case I40E_VSI_FDIR:
9772                 vsi->alloc_queue_pairs = 1;
9773                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
9774                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
9775                 vsi->num_q_vectors = pf->num_fdsb_msix;
9776                 break;
9777
9778         case I40E_VSI_VMDQ2:
9779                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
9780                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
9781                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
9782                 vsi->num_q_vectors = pf->num_vmdq_msix;
9783                 break;
9784
9785         case I40E_VSI_SRIOV:
9786                 vsi->alloc_queue_pairs = pf->num_vf_qps;
9787                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
9788                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
9789                 break;
9790
9791         default:
9792                 WARN_ON(1);
9793                 return -ENODATA;
9794         }
9795
9796         return 0;
9797 }
9798
9799 /**
9800  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
9801  * @vsi: VSI pointer
9802  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
9803  *
9804  * On error: returns error code (negative)
9805  * On success: returns 0
9806  **/
9807 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
9808 {
9809         struct i40e_ring **next_rings;
9810         int size;
9811         int ret = 0;
9812
9813         /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
9814         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
9815                (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
9816         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
9817         if (!vsi->tx_rings)
9818                 return -ENOMEM;
9819         next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
9820         if (i40e_enabled_xdp_vsi(vsi)) {
9821                 vsi->xdp_rings = next_rings;
9822                 next_rings += vsi->alloc_queue_pairs;
9823         }
9824         vsi->rx_rings = next_rings;
9825
9826         if (alloc_qvectors) {
9827                 /* allocate memory for q_vector pointers */
9828                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
9829                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
9830                 if (!vsi->q_vectors) {
9831                         ret = -ENOMEM;
9832                         goto err_vectors;
9833                 }
9834         }
9835         return ret;
9836
9837 err_vectors:
9838         kfree(vsi->tx_rings);
9839         return ret;
9840 }
9841
9842 /**
9843  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
9844  * @pf: board private structure
9845  * @type: type of VSI
9846  *
9847  * On error: returns error code (negative)
9848  * On success: returns vsi index in PF (positive)
9849  **/
9850 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
9851 {
9852         int ret = -ENODEV;
9853         struct i40e_vsi *vsi;
9854         int vsi_idx;
9855         int i;
9856
9857         /* Need to protect the allocation of the VSIs at the PF level */
9858         mutex_lock(&pf->switch_mutex);
9859
9860         /* VSI list may be fragmented if VSI creation/destruction has
9861          * been happening.  We can afford to do a quick scan to look
9862          * for any free VSIs in the list.
9863          *
9864          * find next empty vsi slot, looping back around if necessary
9865          */
9866         i = pf->next_vsi;
9867         while (i < pf->num_alloc_vsi && pf->vsi[i])
9868                 i++;
9869         if (i >= pf->num_alloc_vsi) {
9870                 i = 0;
9871                 while (i < pf->next_vsi && pf->vsi[i])
9872                         i++;
9873         }
9874
9875         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
9876                 vsi_idx = i;             /* Found one! */
9877         } else {
9878                 ret = -ENODEV;
9879                 goto unlock_pf;  /* out of VSI slots! */
9880         }
9881         pf->next_vsi = ++i;
9882
9883         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
9884         if (!vsi) {
9885                 ret = -ENOMEM;
9886                 goto unlock_pf;
9887         }
9888         vsi->type = type;
9889         vsi->back = pf;
9890         set_bit(__I40E_VSI_DOWN, vsi->state);
9891         vsi->flags = 0;
9892         vsi->idx = vsi_idx;
9893         vsi->int_rate_limit = 0;
9894         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
9895                                 pf->rss_table_size : 64;
9896         vsi->netdev_registered = false;
9897         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
9898         hash_init(vsi->mac_filter_hash);
9899         vsi->irqs_ready = false;
9900
9901         ret = i40e_set_num_rings_in_vsi(vsi);
9902         if (ret)
9903                 goto err_rings;
9904
9905         ret = i40e_vsi_alloc_arrays(vsi, true);
9906         if (ret)
9907                 goto err_rings;
9908
9909         /* Setup default MSIX irq handler for VSI */
9910         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
9911
9912         /* Initialize VSI lock */
9913         spin_lock_init(&vsi->mac_filter_hash_lock);
9914         pf->vsi[vsi_idx] = vsi;
9915         ret = vsi_idx;
9916         goto unlock_pf;
9917
9918 err_rings:
9919         pf->next_vsi = i - 1;
9920         kfree(vsi);
9921 unlock_pf:
9922         mutex_unlock(&pf->switch_mutex);
9923         return ret;
9924 }
9925
9926 /**
9927  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
9928  * @type: VSI pointer
9929  * @free_qvectors: a bool to specify if q_vectors need to be freed.
9930  *
9931  * On error: returns error code (negative)
9932  * On success: returns 0
9933  **/
9934 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
9935 {
9936         /* free the ring and vector containers */
9937         if (free_qvectors) {
9938                 kfree(vsi->q_vectors);
9939                 vsi->q_vectors = NULL;
9940         }
9941         kfree(vsi->tx_rings);
9942         vsi->tx_rings = NULL;
9943         vsi->rx_rings = NULL;
9944         vsi->xdp_rings = NULL;
9945 }
9946
9947 /**
9948  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
9949  * and lookup table
9950  * @vsi: Pointer to VSI structure
9951  */
9952 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
9953 {
9954         if (!vsi)
9955                 return;
9956
9957         kfree(vsi->rss_hkey_user);
9958         vsi->rss_hkey_user = NULL;
9959
9960         kfree(vsi->rss_lut_user);
9961         vsi->rss_lut_user = NULL;
9962 }
9963
9964 /**
9965  * i40e_vsi_clear - Deallocate the VSI provided
9966  * @vsi: the VSI being un-configured
9967  **/
9968 static int i40e_vsi_clear(struct i40e_vsi *vsi)
9969 {
9970         struct i40e_pf *pf;
9971
9972         if (!vsi)
9973                 return 0;
9974
9975         if (!vsi->back)
9976                 goto free_vsi;
9977         pf = vsi->back;
9978
9979         mutex_lock(&pf->switch_mutex);
9980         if (!pf->vsi[vsi->idx]) {
9981                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
9982                         vsi->idx, vsi->idx, vsi, vsi->type);
9983                 goto unlock_vsi;
9984         }
9985
9986         if (pf->vsi[vsi->idx] != vsi) {
9987                 dev_err(&pf->pdev->dev,
9988                         "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
9989                         pf->vsi[vsi->idx]->idx,
9990                         pf->vsi[vsi->idx],
9991                         pf->vsi[vsi->idx]->type,
9992                         vsi->idx, vsi, vsi->type);
9993                 goto unlock_vsi;
9994         }
9995
9996         /* updates the PF for this cleared vsi */
9997         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
9998         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
9999
10000         i40e_vsi_free_arrays(vsi, true);
10001         i40e_clear_rss_config_user(vsi);
10002
10003         pf->vsi[vsi->idx] = NULL;
10004         if (vsi->idx < pf->next_vsi)
10005                 pf->next_vsi = vsi->idx;
10006
10007 unlock_vsi:
10008         mutex_unlock(&pf->switch_mutex);
10009 free_vsi:
10010         kfree(vsi);
10011
10012         return 0;
10013 }
10014
10015 /**
10016  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
10017  * @vsi: the VSI being cleaned
10018  **/
10019 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
10020 {
10021         int i;
10022
10023         if (vsi->tx_rings && vsi->tx_rings[0]) {
10024                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
10025                         kfree_rcu(vsi->tx_rings[i], rcu);
10026                         vsi->tx_rings[i] = NULL;
10027                         vsi->rx_rings[i] = NULL;
10028                         if (vsi->xdp_rings)
10029                                 vsi->xdp_rings[i] = NULL;
10030                 }
10031         }
10032 }
10033
10034 /**
10035  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
10036  * @vsi: the VSI being configured
10037  **/
10038 static int i40e_alloc_rings(struct i40e_vsi *vsi)
10039 {
10040         int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
10041         struct i40e_pf *pf = vsi->back;
10042         struct i40e_ring *ring;
10043
10044         /* Set basic values in the rings to be used later during open() */
10045         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
10046                 /* allocate space for both Tx and Rx in one shot */
10047                 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
10048                 if (!ring)
10049                         goto err_out;
10050
10051                 ring->queue_index = i;
10052                 ring->reg_idx = vsi->base_queue + i;
10053                 ring->ring_active = false;
10054                 ring->vsi = vsi;
10055                 ring->netdev = vsi->netdev;
10056                 ring->dev = &pf->pdev->dev;
10057                 ring->count = vsi->num_desc;
10058                 ring->size = 0;
10059                 ring->dcb_tc = 0;
10060                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
10061                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
10062                 ring->tx_itr_setting = pf->tx_itr_default;
10063                 vsi->tx_rings[i] = ring++;
10064
10065                 if (!i40e_enabled_xdp_vsi(vsi))
10066                         goto setup_rx;
10067
10068                 ring->queue_index = vsi->alloc_queue_pairs + i;
10069                 ring->reg_idx = vsi->base_queue + ring->queue_index;
10070                 ring->ring_active = false;
10071                 ring->vsi = vsi;
10072                 ring->netdev = NULL;
10073                 ring->dev = &pf->pdev->dev;
10074                 ring->count = vsi->num_desc;
10075                 ring->size = 0;
10076                 ring->dcb_tc = 0;
10077                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
10078                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
10079                 set_ring_xdp(ring);
10080                 ring->tx_itr_setting = pf->tx_itr_default;
10081                 vsi->xdp_rings[i] = ring++;
10082
10083 setup_rx:
10084                 ring->queue_index = i;
10085                 ring->reg_idx = vsi->base_queue + i;
10086                 ring->ring_active = false;
10087                 ring->vsi = vsi;
10088                 ring->netdev = vsi->netdev;
10089                 ring->dev = &pf->pdev->dev;
10090                 ring->count = vsi->num_desc;
10091                 ring->size = 0;
10092                 ring->dcb_tc = 0;
10093                 ring->rx_itr_setting = pf->rx_itr_default;
10094                 vsi->rx_rings[i] = ring;
10095         }
10096
10097         return 0;
10098
10099 err_out:
10100         i40e_vsi_clear_rings(vsi);
10101         return -ENOMEM;
10102 }
10103
10104 /**
10105  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
10106  * @pf: board private structure
10107  * @vectors: the number of MSI-X vectors to request
10108  *
10109  * Returns the number of vectors reserved, or error
10110  **/
10111 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
10112 {
10113         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
10114                                         I40E_MIN_MSIX, vectors);
10115         if (vectors < 0) {
10116                 dev_info(&pf->pdev->dev,
10117                          "MSI-X vector reservation failed: %d\n", vectors);
10118                 vectors = 0;
10119         }
10120
10121         return vectors;
10122 }
10123
10124 /**
10125  * i40e_init_msix - Setup the MSIX capability
10126  * @pf: board private structure
10127  *
10128  * Work with the OS to set up the MSIX vectors needed.
10129  *
10130  * Returns the number of vectors reserved or negative on failure
10131  **/
10132 static int i40e_init_msix(struct i40e_pf *pf)
10133 {
10134         struct i40e_hw *hw = &pf->hw;
10135         int cpus, extra_vectors;
10136         int vectors_left;
10137         int v_budget, i;
10138         int v_actual;
10139         int iwarp_requested = 0;
10140
10141         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
10142                 return -ENODEV;
10143
10144         /* The number of vectors we'll request will be comprised of:
10145          *   - Add 1 for "other" cause for Admin Queue events, etc.
10146          *   - The number of LAN queue pairs
10147          *      - Queues being used for RSS.
10148          *              We don't need as many as max_rss_size vectors.
10149          *              use rss_size instead in the calculation since that
10150          *              is governed by number of cpus in the system.
10151          *      - assumes symmetric Tx/Rx pairing
10152          *   - The number of VMDq pairs
10153          *   - The CPU count within the NUMA node if iWARP is enabled
10154          * Once we count this up, try the request.
10155          *
10156          * If we can't get what we want, we'll simplify to nearly nothing
10157          * and try again.  If that still fails, we punt.
10158          */
10159         vectors_left = hw->func_caps.num_msix_vectors;
10160         v_budget = 0;
10161
10162         /* reserve one vector for miscellaneous handler */
10163         if (vectors_left) {
10164                 v_budget++;
10165                 vectors_left--;
10166         }
10167
10168         /* reserve some vectors for the main PF traffic queues. Initially we
10169          * only reserve at most 50% of the available vectors, in the case that
10170          * the number of online CPUs is large. This ensures that we can enable
10171          * extra features as well. Once we've enabled the other features, we
10172          * will use any remaining vectors to reach as close as we can to the
10173          * number of online CPUs.
10174          */
10175         cpus = num_online_cpus();
10176         pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
10177         vectors_left -= pf->num_lan_msix;
10178
10179         /* reserve one vector for sideband flow director */
10180         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
10181                 if (vectors_left) {
10182                         pf->num_fdsb_msix = 1;
10183                         v_budget++;
10184                         vectors_left--;
10185                 } else {
10186                         pf->num_fdsb_msix = 0;
10187                 }
10188         }
10189
10190         /* can we reserve enough for iWARP? */
10191         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
10192                 iwarp_requested = pf->num_iwarp_msix;
10193
10194                 if (!vectors_left)
10195                         pf->num_iwarp_msix = 0;
10196                 else if (vectors_left < pf->num_iwarp_msix)
10197                         pf->num_iwarp_msix = 1;
10198                 v_budget += pf->num_iwarp_msix;
10199                 vectors_left -= pf->num_iwarp_msix;
10200         }
10201
10202         /* any vectors left over go for VMDq support */
10203         if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
10204                 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps;
10205                 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted);
10206
10207                 if (!vectors_left) {
10208                         pf->num_vmdq_msix = 0;
10209                         pf->num_vmdq_qps = 0;
10210                 } else {
10211                         /* if we're short on vectors for what's desired, we limit
10212                          * the queues per vmdq.  If this is still more than are
10213                          * available, the user will need to change the number of
10214                          * queues/vectors used by the PF later with the ethtool
10215                          * channels command
10216                          */
10217                         if (vmdq_vecs < vmdq_vecs_wanted)
10218                                 pf->num_vmdq_qps = 1;
10219                         pf->num_vmdq_msix = pf->num_vmdq_qps;
10220
10221                         v_budget += vmdq_vecs;
10222                         vectors_left -= vmdq_vecs;
10223                 }
10224         }
10225
10226         /* On systems with a large number of SMP cores, we previously limited
10227          * the number of vectors for num_lan_msix to be at most 50% of the
10228          * available vectors, to allow for other features. Now, we add back
10229          * the remaining vectors. However, we ensure that the total
10230          * num_lan_msix will not exceed num_online_cpus(). To do this, we
10231          * calculate the number of vectors we can add without going over the
10232          * cap of CPUs. For systems with a small number of CPUs this will be
10233          * zero.
10234          */
10235         extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
10236         pf->num_lan_msix += extra_vectors;
10237         vectors_left -= extra_vectors;
10238
10239         WARN(vectors_left < 0,
10240              "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
10241
10242         v_budget += pf->num_lan_msix;
10243         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
10244                                    GFP_KERNEL);
10245         if (!pf->msix_entries)
10246                 return -ENOMEM;
10247
10248         for (i = 0; i < v_budget; i++)
10249                 pf->msix_entries[i].entry = i;
10250         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
10251
10252         if (v_actual < I40E_MIN_MSIX) {
10253                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
10254                 kfree(pf->msix_entries);
10255                 pf->msix_entries = NULL;
10256                 pci_disable_msix(pf->pdev);
10257                 return -ENODEV;
10258
10259         } else if (v_actual == I40E_MIN_MSIX) {
10260                 /* Adjust for minimal MSIX use */
10261                 pf->num_vmdq_vsis = 0;
10262                 pf->num_vmdq_qps = 0;
10263                 pf->num_lan_qps = 1;
10264                 pf->num_lan_msix = 1;
10265
10266         } else if (v_actual != v_budget) {
10267                 /* If we have limited resources, we will start with no vectors
10268                  * for the special features and then allocate vectors to some
10269                  * of these features based on the policy and at the end disable
10270                  * the features that did not get any vectors.
10271                  */
10272                 int vec;
10273
10274                 dev_info(&pf->pdev->dev,
10275                          "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
10276                          v_actual, v_budget);
10277                 /* reserve the misc vector */
10278                 vec = v_actual - 1;
10279
10280                 /* Scale vector usage down */
10281                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
10282                 pf->num_vmdq_vsis = 1;
10283                 pf->num_vmdq_qps = 1;
10284
10285                 /* partition out the remaining vectors */
10286                 switch (vec) {
10287                 case 2:
10288                         pf->num_lan_msix = 1;
10289                         break;
10290                 case 3:
10291                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
10292                                 pf->num_lan_msix = 1;
10293                                 pf->num_iwarp_msix = 1;
10294                         } else {
10295                                 pf->num_lan_msix = 2;
10296                         }
10297                         break;
10298                 default:
10299                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
10300                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
10301                                                  iwarp_requested);
10302                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
10303                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
10304                         } else {
10305                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
10306                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
10307                         }
10308                         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
10309                                 pf->num_fdsb_msix = 1;
10310                                 vec--;
10311                         }
10312                         pf->num_lan_msix = min_t(int,
10313                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
10314                                                               pf->num_lan_msix);
10315                         pf->num_lan_qps = pf->num_lan_msix;
10316                         break;
10317                 }
10318         }
10319
10320         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
10321             (pf->num_fdsb_msix == 0)) {
10322                 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
10323                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
10324                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
10325         }
10326         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
10327             (pf->num_vmdq_msix == 0)) {
10328                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
10329                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
10330         }
10331
10332         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
10333             (pf->num_iwarp_msix == 0)) {
10334                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
10335                 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
10336         }
10337         i40e_debug(&pf->hw, I40E_DEBUG_INIT,
10338                    "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
10339                    pf->num_lan_msix,
10340                    pf->num_vmdq_msix * pf->num_vmdq_vsis,
10341                    pf->num_fdsb_msix,
10342                    pf->num_iwarp_msix);
10343
10344         return v_actual;
10345 }
10346
10347 /**
10348  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
10349  * @vsi: the VSI being configured
10350  * @v_idx: index of the vector in the vsi struct
10351  * @cpu: cpu to be used on affinity_mask
10352  *
10353  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
10354  **/
10355 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu)
10356 {
10357         struct i40e_q_vector *q_vector;
10358
10359         /* allocate q_vector */
10360         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
10361         if (!q_vector)
10362                 return -ENOMEM;
10363
10364         q_vector->vsi = vsi;
10365         q_vector->v_idx = v_idx;
10366         cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
10367
10368         if (vsi->netdev)
10369                 netif_napi_add(vsi->netdev, &q_vector->napi,
10370                                i40e_napi_poll, NAPI_POLL_WEIGHT);
10371
10372         q_vector->rx.latency_range = I40E_LOW_LATENCY;
10373         q_vector->tx.latency_range = I40E_LOW_LATENCY;
10374
10375         /* tie q_vector and vsi together */
10376         vsi->q_vectors[v_idx] = q_vector;
10377
10378         return 0;
10379 }
10380
10381 /**
10382  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
10383  * @vsi: the VSI being configured
10384  *
10385  * We allocate one q_vector per queue interrupt.  If allocation fails we
10386  * return -ENOMEM.
10387  **/
10388 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
10389 {
10390         struct i40e_pf *pf = vsi->back;
10391         int err, v_idx, num_q_vectors, current_cpu;
10392
10393         /* if not MSIX, give the one vector only to the LAN VSI */
10394         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
10395                 num_q_vectors = vsi->num_q_vectors;
10396         else if (vsi == pf->vsi[pf->lan_vsi])
10397                 num_q_vectors = 1;
10398         else
10399                 return -EINVAL;
10400
10401         current_cpu = cpumask_first(cpu_online_mask);
10402
10403         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
10404                 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu);
10405                 if (err)
10406                         goto err_out;
10407                 current_cpu = cpumask_next(current_cpu, cpu_online_mask);
10408                 if (unlikely(current_cpu >= nr_cpu_ids))
10409                         current_cpu = cpumask_first(cpu_online_mask);
10410         }
10411
10412         return 0;
10413
10414 err_out:
10415         while (v_idx--)
10416                 i40e_free_q_vector(vsi, v_idx);
10417
10418         return err;
10419 }
10420
10421 /**
10422  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
10423  * @pf: board private structure to initialize
10424  **/
10425 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
10426 {
10427         int vectors = 0;
10428         ssize_t size;
10429
10430         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
10431                 vectors = i40e_init_msix(pf);
10432                 if (vectors < 0) {
10433                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
10434                                        I40E_FLAG_IWARP_ENABLED  |
10435                                        I40E_FLAG_RSS_ENABLED    |
10436                                        I40E_FLAG_DCB_CAPABLE    |
10437                                        I40E_FLAG_DCB_ENABLED    |
10438                                        I40E_FLAG_SRIOV_ENABLED  |
10439                                        I40E_FLAG_FD_SB_ENABLED  |
10440                                        I40E_FLAG_FD_ATR_ENABLED |
10441                                        I40E_FLAG_VMDQ_ENABLED);
10442                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
10443
10444                         /* rework the queue expectations without MSIX */
10445                         i40e_determine_queue_usage(pf);
10446                 }
10447         }
10448
10449         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
10450             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
10451                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
10452                 vectors = pci_enable_msi(pf->pdev);
10453                 if (vectors < 0) {
10454                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
10455                                  vectors);
10456                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
10457                 }
10458                 vectors = 1;  /* one MSI or Legacy vector */
10459         }
10460
10461         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
10462                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
10463
10464         /* set up vector assignment tracking */
10465         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
10466         pf->irq_pile = kzalloc(size, GFP_KERNEL);
10467         if (!pf->irq_pile) {
10468                 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n");
10469                 return -ENOMEM;
10470         }
10471         pf->irq_pile->num_entries = vectors;
10472         pf->irq_pile->search_hint = 0;
10473
10474         /* track first vector for misc interrupts, ignore return */
10475         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
10476
10477         return 0;
10478 }
10479
10480 /**
10481  * i40e_restore_interrupt_scheme - Restore the interrupt scheme
10482  * @pf: private board data structure
10483  *
10484  * Restore the interrupt scheme that was cleared when we suspended the
10485  * device. This should be called during resume to re-allocate the q_vectors
10486  * and reacquire IRQs.
10487  */
10488 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
10489 {
10490         int err, i;
10491
10492         /* We cleared the MSI and MSI-X flags when disabling the old interrupt
10493          * scheme. We need to re-enabled them here in order to attempt to
10494          * re-acquire the MSI or MSI-X vectors
10495          */
10496         pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
10497
10498         err = i40e_init_interrupt_scheme(pf);
10499         if (err)
10500                 return err;
10501
10502         /* Now that we've re-acquired IRQs, we need to remap the vectors and
10503          * rings together again.
10504          */
10505         for (i = 0; i < pf->num_alloc_vsi; i++) {
10506                 if (pf->vsi[i]) {
10507                         err = i40e_vsi_alloc_q_vectors(pf->vsi[i]);
10508                         if (err)
10509                                 goto err_unwind;
10510                         i40e_vsi_map_rings_to_vectors(pf->vsi[i]);
10511                 }
10512         }
10513
10514         err = i40e_setup_misc_vector(pf);
10515         if (err)
10516                 goto err_unwind;
10517
10518         return 0;
10519
10520 err_unwind:
10521         while (i--) {
10522                 if (pf->vsi[i])
10523                         i40e_vsi_free_q_vectors(pf->vsi[i]);
10524         }
10525
10526         return err;
10527 }
10528
10529 /**
10530  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
10531  * @pf: board private structure
10532  *
10533  * This sets up the handler for MSIX 0, which is used to manage the
10534  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
10535  * when in MSI or Legacy interrupt mode.
10536  **/
10537 static int i40e_setup_misc_vector(struct i40e_pf *pf)
10538 {
10539         struct i40e_hw *hw = &pf->hw;
10540         int err = 0;
10541
10542         /* Only request the IRQ once, the first time through. */
10543         if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
10544                 err = request_irq(pf->msix_entries[0].vector,
10545                                   i40e_intr, 0, pf->int_name, pf);
10546                 if (err) {
10547                         clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
10548                         dev_info(&pf->pdev->dev,
10549                                  "request_irq for %s failed: %d\n",
10550                                  pf->int_name, err);
10551                         return -EFAULT;
10552                 }
10553         }
10554
10555         i40e_enable_misc_int_causes(pf);
10556
10557         /* associate no queues to the misc vector */
10558         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
10559         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
10560
10561         i40e_flush(hw);
10562
10563         i40e_irq_dynamic_enable_icr0(pf);
10564
10565         return err;
10566 }
10567
10568 /**
10569  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
10570  * @vsi: Pointer to vsi structure
10571  * @seed: Buffter to store the hash keys
10572  * @lut: Buffer to store the lookup table entries
10573  * @lut_size: Size of buffer to store the lookup table entries
10574  *
10575  * Return 0 on success, negative on failure
10576  */
10577 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
10578                            u8 *lut, u16 lut_size)
10579 {
10580         struct i40e_pf *pf = vsi->back;
10581         struct i40e_hw *hw = &pf->hw;
10582         int ret = 0;
10583
10584         if (seed) {
10585                 ret = i40e_aq_get_rss_key(hw, vsi->id,
10586                         (struct i40e_aqc_get_set_rss_key_data *)seed);
10587                 if (ret) {
10588                         dev_info(&pf->pdev->dev,
10589                                  "Cannot get RSS key, err %s aq_err %s\n",
10590                                  i40e_stat_str(&pf->hw, ret),
10591                                  i40e_aq_str(&pf->hw,
10592                                              pf->hw.aq.asq_last_status));
10593                         return ret;
10594                 }
10595         }
10596
10597         if (lut) {
10598                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
10599
10600                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
10601                 if (ret) {
10602                         dev_info(&pf->pdev->dev,
10603                                  "Cannot get RSS lut, err %s aq_err %s\n",
10604                                  i40e_stat_str(&pf->hw, ret),
10605                                  i40e_aq_str(&pf->hw,
10606                                              pf->hw.aq.asq_last_status));
10607                         return ret;
10608                 }
10609         }
10610
10611         return ret;
10612 }
10613
10614 /**
10615  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
10616  * @vsi: Pointer to vsi structure
10617  * @seed: RSS hash seed
10618  * @lut: Lookup table
10619  * @lut_size: Lookup table size
10620  *
10621  * Returns 0 on success, negative on failure
10622  **/
10623 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
10624                                const u8 *lut, u16 lut_size)
10625 {
10626         struct i40e_pf *pf = vsi->back;
10627         struct i40e_hw *hw = &pf->hw;
10628         u16 vf_id = vsi->vf_id;
10629         u8 i;
10630
10631         /* Fill out hash function seed */
10632         if (seed) {
10633                 u32 *seed_dw = (u32 *)seed;
10634
10635                 if (vsi->type == I40E_VSI_MAIN) {
10636                         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
10637                                 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
10638                 } else if (vsi->type == I40E_VSI_SRIOV) {
10639                         for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
10640                                 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
10641                 } else {
10642                         dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
10643                 }
10644         }
10645
10646         if (lut) {
10647                 u32 *lut_dw = (u32 *)lut;
10648
10649                 if (vsi->type == I40E_VSI_MAIN) {
10650                         if (lut_size != I40E_HLUT_ARRAY_SIZE)
10651                                 return -EINVAL;
10652                         for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
10653                                 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
10654                 } else if (vsi->type == I40E_VSI_SRIOV) {
10655                         if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
10656                                 return -EINVAL;
10657                         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
10658                                 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
10659                 } else {
10660                         dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
10661                 }
10662         }
10663         i40e_flush(hw);
10664
10665         return 0;
10666 }
10667
10668 /**
10669  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
10670  * @vsi: Pointer to VSI structure
10671  * @seed: Buffer to store the keys
10672  * @lut: Buffer to store the lookup table entries
10673  * @lut_size: Size of buffer to store the lookup table entries
10674  *
10675  * Returns 0 on success, negative on failure
10676  */
10677 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
10678                             u8 *lut, u16 lut_size)
10679 {
10680         struct i40e_pf *pf = vsi->back;
10681         struct i40e_hw *hw = &pf->hw;
10682         u16 i;
10683
10684         if (seed) {
10685                 u32 *seed_dw = (u32 *)seed;
10686
10687                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
10688                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
10689         }
10690         if (lut) {
10691                 u32 *lut_dw = (u32 *)lut;
10692
10693                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
10694                         return -EINVAL;
10695                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
10696                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
10697         }
10698
10699         return 0;
10700 }
10701
10702 /**
10703  * i40e_config_rss - Configure RSS keys and lut
10704  * @vsi: Pointer to VSI structure
10705  * @seed: RSS hash seed
10706  * @lut: Lookup table
10707  * @lut_size: Lookup table size
10708  *
10709  * Returns 0 on success, negative on failure
10710  */
10711 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
10712 {
10713         struct i40e_pf *pf = vsi->back;
10714
10715         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
10716                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
10717         else
10718                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
10719 }
10720
10721 /**
10722  * i40e_get_rss - Get RSS keys and lut
10723  * @vsi: Pointer to VSI structure
10724  * @seed: Buffer to store the keys
10725  * @lut: Buffer to store the lookup table entries
10726  * lut_size: Size of buffer to store the lookup table entries
10727  *
10728  * Returns 0 on success, negative on failure
10729  */
10730 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
10731 {
10732         struct i40e_pf *pf = vsi->back;
10733
10734         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
10735                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
10736         else
10737                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
10738 }
10739
10740 /**
10741  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
10742  * @pf: Pointer to board private structure
10743  * @lut: Lookup table
10744  * @rss_table_size: Lookup table size
10745  * @rss_size: Range of queue number for hashing
10746  */
10747 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
10748                        u16 rss_table_size, u16 rss_size)
10749 {
10750         u16 i;
10751
10752         for (i = 0; i < rss_table_size; i++)
10753                 lut[i] = i % rss_size;
10754 }
10755
10756 /**
10757  * i40e_pf_config_rss - Prepare for RSS if used
10758  * @pf: board private structure
10759  **/
10760 static int i40e_pf_config_rss(struct i40e_pf *pf)
10761 {
10762         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10763         u8 seed[I40E_HKEY_ARRAY_SIZE];
10764         u8 *lut;
10765         struct i40e_hw *hw = &pf->hw;
10766         u32 reg_val;
10767         u64 hena;
10768         int ret;
10769
10770         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
10771         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
10772                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
10773         hena |= i40e_pf_get_default_rss_hena(pf);
10774
10775         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
10776         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
10777
10778         /* Determine the RSS table size based on the hardware capabilities */
10779         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
10780         reg_val = (pf->rss_table_size == 512) ?
10781                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
10782                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
10783         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
10784
10785         /* Determine the RSS size of the VSI */
10786         if (!vsi->rss_size) {
10787                 u16 qcount;
10788
10789                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
10790                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
10791         }
10792         if (!vsi->rss_size)
10793                 return -EINVAL;
10794
10795         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
10796         if (!lut)
10797                 return -ENOMEM;
10798
10799         /* Use user configured lut if there is one, otherwise use default */
10800         if (vsi->rss_lut_user)
10801                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
10802         else
10803                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
10804
10805         /* Use user configured hash key if there is one, otherwise
10806          * use default.
10807          */
10808         if (vsi->rss_hkey_user)
10809                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
10810         else
10811                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
10812         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
10813         kfree(lut);
10814
10815         return ret;
10816 }
10817
10818 /**
10819  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
10820  * @pf: board private structure
10821  * @queue_count: the requested queue count for rss.
10822  *
10823  * returns 0 if rss is not enabled, if enabled returns the final rss queue
10824  * count which may be different from the requested queue count.
10825  * Note: expects to be called while under rtnl_lock()
10826  **/
10827 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
10828 {
10829         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10830         int new_rss_size;
10831
10832         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
10833                 return 0;
10834
10835         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
10836
10837         if (queue_count != vsi->num_queue_pairs) {
10838                 u16 qcount;
10839
10840                 vsi->req_queue_pairs = queue_count;
10841                 i40e_prep_for_reset(pf, true);
10842
10843                 pf->alloc_rss_size = new_rss_size;
10844
10845                 i40e_reset_and_rebuild(pf, true, true);
10846
10847                 /* Discard the user configured hash keys and lut, if less
10848                  * queues are enabled.
10849                  */
10850                 if (queue_count < vsi->rss_size) {
10851                         i40e_clear_rss_config_user(vsi);
10852                         dev_dbg(&pf->pdev->dev,
10853                                 "discard user configured hash keys and lut\n");
10854                 }
10855
10856                 /* Reset vsi->rss_size, as number of enabled queues changed */
10857                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
10858                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
10859
10860                 i40e_pf_config_rss(pf);
10861         }
10862         dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
10863                  vsi->req_queue_pairs, pf->rss_size_max);
10864         return pf->alloc_rss_size;
10865 }
10866
10867 /**
10868  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
10869  * @pf: board private structure
10870  **/
10871 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf)
10872 {
10873         i40e_status status;
10874         bool min_valid, max_valid;
10875         u32 max_bw, min_bw;
10876
10877         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
10878                                            &min_valid, &max_valid);
10879
10880         if (!status) {
10881                 if (min_valid)
10882                         pf->min_bw = min_bw;
10883                 if (max_valid)
10884                         pf->max_bw = max_bw;
10885         }
10886
10887         return status;
10888 }
10889
10890 /**
10891  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
10892  * @pf: board private structure
10893  **/
10894 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf)
10895 {
10896         struct i40e_aqc_configure_partition_bw_data bw_data;
10897         i40e_status status;
10898
10899         /* Set the valid bit for this PF */
10900         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
10901         bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
10902         bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
10903
10904         /* Set the new bandwidths */
10905         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
10906
10907         return status;
10908 }
10909
10910 /**
10911  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
10912  * @pf: board private structure
10913  **/
10914 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
10915 {
10916         /* Commit temporary BW setting to permanent NVM image */
10917         enum i40e_admin_queue_err last_aq_status;
10918         i40e_status ret;
10919         u16 nvm_word;
10920
10921         if (pf->hw.partition_id != 1) {
10922                 dev_info(&pf->pdev->dev,
10923                          "Commit BW only works on partition 1! This is partition %d",
10924                          pf->hw.partition_id);
10925                 ret = I40E_NOT_SUPPORTED;
10926                 goto bw_commit_out;
10927         }
10928
10929         /* Acquire NVM for read access */
10930         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
10931         last_aq_status = pf->hw.aq.asq_last_status;
10932         if (ret) {
10933                 dev_info(&pf->pdev->dev,
10934                          "Cannot acquire NVM for read access, err %s aq_err %s\n",
10935                          i40e_stat_str(&pf->hw, ret),
10936                          i40e_aq_str(&pf->hw, last_aq_status));
10937                 goto bw_commit_out;
10938         }
10939
10940         /* Read word 0x10 of NVM - SW compatibility word 1 */
10941         ret = i40e_aq_read_nvm(&pf->hw,
10942                                I40E_SR_NVM_CONTROL_WORD,
10943                                0x10, sizeof(nvm_word), &nvm_word,
10944                                false, NULL);
10945         /* Save off last admin queue command status before releasing
10946          * the NVM
10947          */
10948         last_aq_status = pf->hw.aq.asq_last_status;
10949         i40e_release_nvm(&pf->hw);
10950         if (ret) {
10951                 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
10952                          i40e_stat_str(&pf->hw, ret),
10953                          i40e_aq_str(&pf->hw, last_aq_status));
10954                 goto bw_commit_out;
10955         }
10956
10957         /* Wait a bit for NVM release to complete */
10958         msleep(50);
10959
10960         /* Acquire NVM for write access */
10961         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
10962         last_aq_status = pf->hw.aq.asq_last_status;
10963         if (ret) {
10964                 dev_info(&pf->pdev->dev,
10965                          "Cannot acquire NVM for write access, err %s aq_err %s\n",
10966                          i40e_stat_str(&pf->hw, ret),
10967                          i40e_aq_str(&pf->hw, last_aq_status));
10968                 goto bw_commit_out;
10969         }
10970         /* Write it back out unchanged to initiate update NVM,
10971          * which will force a write of the shadow (alt) RAM to
10972          * the NVM - thus storing the bandwidth values permanently.
10973          */
10974         ret = i40e_aq_update_nvm(&pf->hw,
10975                                  I40E_SR_NVM_CONTROL_WORD,
10976                                  0x10, sizeof(nvm_word),
10977                                  &nvm_word, true, NULL);
10978         /* Save off last admin queue command status before releasing
10979          * the NVM
10980          */
10981         last_aq_status = pf->hw.aq.asq_last_status;
10982         i40e_release_nvm(&pf->hw);
10983         if (ret)
10984                 dev_info(&pf->pdev->dev,
10985                          "BW settings NOT SAVED, err %s aq_err %s\n",
10986                          i40e_stat_str(&pf->hw, ret),
10987                          i40e_aq_str(&pf->hw, last_aq_status));
10988 bw_commit_out:
10989
10990         return ret;
10991 }
10992
10993 /**
10994  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
10995  * @pf: board private structure to initialize
10996  *
10997  * i40e_sw_init initializes the Adapter private data structure.
10998  * Fields are initialized based on PCI device information and
10999  * OS network device settings (MTU size).
11000  **/
11001 static int i40e_sw_init(struct i40e_pf *pf)
11002 {
11003         int err = 0;
11004         int size;
11005
11006         /* Set default capability flags */
11007         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
11008                     I40E_FLAG_MSI_ENABLED     |
11009                     I40E_FLAG_MSIX_ENABLED;
11010
11011         /* Set default ITR */
11012         pf->rx_itr_default = I40E_ITR_RX_DEF;
11013         pf->tx_itr_default = I40E_ITR_TX_DEF;
11014
11015         /* Depending on PF configurations, it is possible that the RSS
11016          * maximum might end up larger than the available queues
11017          */
11018         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
11019         pf->alloc_rss_size = 1;
11020         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
11021         pf->rss_size_max = min_t(int, pf->rss_size_max,
11022                                  pf->hw.func_caps.num_tx_qp);
11023         if (pf->hw.func_caps.rss) {
11024                 pf->flags |= I40E_FLAG_RSS_ENABLED;
11025                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
11026                                            num_online_cpus());
11027         }
11028
11029         /* MFP mode enabled */
11030         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
11031                 pf->flags |= I40E_FLAG_MFP_ENABLED;
11032                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
11033                 if (i40e_get_partition_bw_setting(pf)) {
11034                         dev_warn(&pf->pdev->dev,
11035                                  "Could not get partition bw settings\n");
11036                 } else {
11037                         dev_info(&pf->pdev->dev,
11038                                  "Partition BW Min = %8.8x, Max = %8.8x\n",
11039                                  pf->min_bw, pf->max_bw);
11040
11041                         /* nudge the Tx scheduler */
11042                         i40e_set_partition_bw_setting(pf);
11043                 }
11044         }
11045
11046         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
11047             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
11048                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
11049                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
11050                 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
11051                     pf->hw.num_partitions > 1)
11052                         dev_info(&pf->pdev->dev,
11053                                  "Flow Director Sideband mode Disabled in MFP mode\n");
11054                 else
11055                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
11056                 pf->fdir_pf_filter_count =
11057                                  pf->hw.func_caps.fd_filters_guaranteed;
11058                 pf->hw.fdir_shared_filter_count =
11059                                  pf->hw.func_caps.fd_filters_best_effort;
11060         }
11061
11062         if (pf->hw.mac.type == I40E_MAC_X722) {
11063                 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
11064                                     I40E_HW_128_QP_RSS_CAPABLE |
11065                                     I40E_HW_ATR_EVICT_CAPABLE |
11066                                     I40E_HW_WB_ON_ITR_CAPABLE |
11067                                     I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
11068                                     I40E_HW_NO_PCI_LINK_CHECK |
11069                                     I40E_HW_USE_SET_LLDP_MIB |
11070                                     I40E_HW_GENEVE_OFFLOAD_CAPABLE |
11071                                     I40E_HW_PTP_L4_CAPABLE |
11072                                     I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
11073                                     I40E_HW_OUTER_UDP_CSUM_CAPABLE);
11074
11075 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
11076                 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
11077                     I40E_FDEVICT_PCTYPE_DEFAULT) {
11078                         dev_warn(&pf->pdev->dev,
11079                                  "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
11080                         pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
11081                 }
11082         } else if ((pf->hw.aq.api_maj_ver > 1) ||
11083                    ((pf->hw.aq.api_maj_ver == 1) &&
11084                     (pf->hw.aq.api_min_ver > 4))) {
11085                 /* Supported in FW API version higher than 1.4 */
11086                 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
11087         }
11088
11089         /* Enable HW ATR eviction if possible */
11090         if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
11091                 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
11092
11093         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
11094             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
11095             (pf->hw.aq.fw_maj_ver < 4))) {
11096                 pf->hw_features |= I40E_HW_RESTART_AUTONEG;
11097                 /* No DCB support  for FW < v4.33 */
11098                 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
11099         }
11100
11101         /* Disable FW LLDP if FW < v4.3 */
11102         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
11103             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
11104             (pf->hw.aq.fw_maj_ver < 4)))
11105                 pf->hw_features |= I40E_HW_STOP_FW_LLDP;
11106
11107         /* Use the FW Set LLDP MIB API if FW > v4.40 */
11108         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
11109             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
11110             (pf->hw.aq.fw_maj_ver >= 5)))
11111                 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
11112
11113         /* Enable PTP L4 if FW > v6.0 */
11114         if (pf->hw.mac.type == I40E_MAC_XL710 &&
11115             pf->hw.aq.fw_maj_ver >= 6)
11116                 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
11117
11118         if (pf->hw.func_caps.vmdq) {
11119                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
11120                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
11121                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
11122         }
11123
11124         if (pf->hw.func_caps.iwarp) {
11125                 pf->flags |= I40E_FLAG_IWARP_ENABLED;
11126                 /* IWARP needs one extra vector for CQP just like MISC.*/
11127                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
11128         }
11129
11130 #ifdef CONFIG_PCI_IOV
11131         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
11132                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
11133                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
11134                 pf->num_req_vfs = min_t(int,
11135                                         pf->hw.func_caps.num_vfs,
11136                                         I40E_MAX_VF_COUNT);
11137         }
11138 #endif /* CONFIG_PCI_IOV */
11139         pf->eeprom_version = 0xDEAD;
11140         pf->lan_veb = I40E_NO_VEB;
11141         pf->lan_vsi = I40E_NO_VSI;
11142
11143         /* By default FW has this off for performance reasons */
11144         pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
11145
11146         /* set up queue assignment tracking */
11147         size = sizeof(struct i40e_lump_tracking)
11148                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
11149         pf->qp_pile = kzalloc(size, GFP_KERNEL);
11150         if (!pf->qp_pile) {
11151                 err = -ENOMEM;
11152                 goto sw_init_done;
11153         }
11154         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
11155         pf->qp_pile->search_hint = 0;
11156
11157         pf->tx_timeout_recovery_level = 1;
11158
11159         mutex_init(&pf->switch_mutex);
11160
11161 sw_init_done:
11162         return err;
11163 }
11164
11165 /**
11166  * i40e_set_ntuple - set the ntuple feature flag and take action
11167  * @pf: board private structure to initialize
11168  * @features: the feature set that the stack is suggesting
11169  *
11170  * returns a bool to indicate if reset needs to happen
11171  **/
11172 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
11173 {
11174         bool need_reset = false;
11175
11176         /* Check if Flow Director n-tuple support was enabled or disabled.  If
11177          * the state changed, we need to reset.
11178          */
11179         if (features & NETIF_F_NTUPLE) {
11180                 /* Enable filters and mark for reset */
11181                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
11182                         need_reset = true;
11183                 /* enable FD_SB only if there is MSI-X vector and no cloud
11184                  * filters exist
11185                  */
11186                 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
11187                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
11188                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
11189                 }
11190         } else {
11191                 /* turn off filters, mark for reset and clear SW filter list */
11192                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11193                         need_reset = true;
11194                         i40e_fdir_filter_exit(pf);
11195                 }
11196                 pf->flags &= ~(I40E_FLAG_FD_SB_ENABLED |
11197                                I40E_FLAG_FD_SB_AUTO_DISABLED);
11198                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11199
11200                 /* reset fd counters */
11201                 pf->fd_add_err = 0;
11202                 pf->fd_atr_cnt = 0;
11203                 /* if ATR was auto disabled it can be re-enabled. */
11204                 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) {
11205                         pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
11206                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
11207                             (I40E_DEBUG_FD & pf->hw.debug_mask))
11208                                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
11209                 }
11210         }
11211         return need_reset;
11212 }
11213
11214 /**
11215  * i40e_clear_rss_lut - clear the rx hash lookup table
11216  * @vsi: the VSI being configured
11217  **/
11218 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
11219 {
11220         struct i40e_pf *pf = vsi->back;
11221         struct i40e_hw *hw = &pf->hw;
11222         u16 vf_id = vsi->vf_id;
11223         u8 i;
11224
11225         if (vsi->type == I40E_VSI_MAIN) {
11226                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
11227                         wr32(hw, I40E_PFQF_HLUT(i), 0);
11228         } else if (vsi->type == I40E_VSI_SRIOV) {
11229                 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
11230                         i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
11231         } else {
11232                 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
11233         }
11234 }
11235
11236 /**
11237  * i40e_set_features - set the netdev feature flags
11238  * @netdev: ptr to the netdev being adjusted
11239  * @features: the feature set that the stack is suggesting
11240  * Note: expects to be called while under rtnl_lock()
11241  **/
11242 static int i40e_set_features(struct net_device *netdev,
11243                              netdev_features_t features)
11244 {
11245         struct i40e_netdev_priv *np = netdev_priv(netdev);
11246         struct i40e_vsi *vsi = np->vsi;
11247         struct i40e_pf *pf = vsi->back;
11248         bool need_reset;
11249
11250         if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
11251                 i40e_pf_config_rss(pf);
11252         else if (!(features & NETIF_F_RXHASH) &&
11253                  netdev->features & NETIF_F_RXHASH)
11254                 i40e_clear_rss_lut(vsi);
11255
11256         if (features & NETIF_F_HW_VLAN_CTAG_RX)
11257                 i40e_vlan_stripping_enable(vsi);
11258         else
11259                 i40e_vlan_stripping_disable(vsi);
11260
11261         if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
11262                 dev_err(&pf->pdev->dev,
11263                         "Offloaded tc filters active, can't turn hw_tc_offload off");
11264                 return -EINVAL;
11265         }
11266
11267         need_reset = i40e_set_ntuple(pf, features);
11268
11269         if (need_reset)
11270                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
11271
11272         return 0;
11273 }
11274
11275 /**
11276  * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port
11277  * @pf: board private structure
11278  * @port: The UDP port to look up
11279  *
11280  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
11281  **/
11282 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port)
11283 {
11284         u8 i;
11285
11286         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
11287                 if (pf->udp_ports[i].port == port)
11288                         return i;
11289         }
11290
11291         return i;
11292 }
11293
11294 /**
11295  * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up
11296  * @netdev: This physical port's netdev
11297  * @ti: Tunnel endpoint information
11298  **/
11299 static void i40e_udp_tunnel_add(struct net_device *netdev,
11300                                 struct udp_tunnel_info *ti)
11301 {
11302         struct i40e_netdev_priv *np = netdev_priv(netdev);
11303         struct i40e_vsi *vsi = np->vsi;
11304         struct i40e_pf *pf = vsi->back;
11305         u16 port = ntohs(ti->port);
11306         u8 next_idx;
11307         u8 idx;
11308
11309         idx = i40e_get_udp_port_idx(pf, port);
11310
11311         /* Check if port already exists */
11312         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
11313                 netdev_info(netdev, "port %d already offloaded\n", port);
11314                 return;
11315         }
11316
11317         /* Now check if there is space to add the new port */
11318         next_idx = i40e_get_udp_port_idx(pf, 0);
11319
11320         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
11321                 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n",
11322                             port);
11323                 return;
11324         }
11325
11326         switch (ti->type) {
11327         case UDP_TUNNEL_TYPE_VXLAN:
11328                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
11329                 break;
11330         case UDP_TUNNEL_TYPE_GENEVE:
11331                 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE))
11332                         return;
11333                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
11334                 break;
11335         default:
11336                 return;
11337         }
11338
11339         /* New port: add it and mark its index in the bitmap */
11340         pf->udp_ports[next_idx].port = port;
11341         pf->pending_udp_bitmap |= BIT_ULL(next_idx);
11342         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
11343 }
11344
11345 /**
11346  * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away
11347  * @netdev: This physical port's netdev
11348  * @ti: Tunnel endpoint information
11349  **/
11350 static void i40e_udp_tunnel_del(struct net_device *netdev,
11351                                 struct udp_tunnel_info *ti)
11352 {
11353         struct i40e_netdev_priv *np = netdev_priv(netdev);
11354         struct i40e_vsi *vsi = np->vsi;
11355         struct i40e_pf *pf = vsi->back;
11356         u16 port = ntohs(ti->port);
11357         u8 idx;
11358
11359         idx = i40e_get_udp_port_idx(pf, port);
11360
11361         /* Check if port already exists */
11362         if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS)
11363                 goto not_found;
11364
11365         switch (ti->type) {
11366         case UDP_TUNNEL_TYPE_VXLAN:
11367                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN)
11368                         goto not_found;
11369                 break;
11370         case UDP_TUNNEL_TYPE_GENEVE:
11371                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE)
11372                         goto not_found;
11373                 break;
11374         default:
11375                 goto not_found;
11376         }
11377
11378         /* if port exists, set it to 0 (mark for deletion)
11379          * and make it pending
11380          */
11381         pf->udp_ports[idx].port = 0;
11382         pf->pending_udp_bitmap |= BIT_ULL(idx);
11383         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
11384
11385         return;
11386 not_found:
11387         netdev_warn(netdev, "UDP port %d was not found, not deleting\n",
11388                     port);
11389 }
11390
11391 static int i40e_get_phys_port_id(struct net_device *netdev,
11392                                  struct netdev_phys_item_id *ppid)
11393 {
11394         struct i40e_netdev_priv *np = netdev_priv(netdev);
11395         struct i40e_pf *pf = np->vsi->back;
11396         struct i40e_hw *hw = &pf->hw;
11397
11398         if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
11399                 return -EOPNOTSUPP;
11400
11401         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
11402         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
11403
11404         return 0;
11405 }
11406
11407 /**
11408  * i40e_ndo_fdb_add - add an entry to the hardware database
11409  * @ndm: the input from the stack
11410  * @tb: pointer to array of nladdr (unused)
11411  * @dev: the net device pointer
11412  * @addr: the MAC address entry being added
11413  * @flags: instructions from stack about fdb operation
11414  */
11415 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
11416                             struct net_device *dev,
11417                             const unsigned char *addr, u16 vid,
11418                             u16 flags)
11419 {
11420         struct i40e_netdev_priv *np = netdev_priv(dev);
11421         struct i40e_pf *pf = np->vsi->back;
11422         int err = 0;
11423
11424         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
11425                 return -EOPNOTSUPP;
11426
11427         if (vid) {
11428                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
11429                 return -EINVAL;
11430         }
11431
11432         /* Hardware does not support aging addresses so if a
11433          * ndm_state is given only allow permanent addresses
11434          */
11435         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
11436                 netdev_info(dev, "FDB only supports static addresses\n");
11437                 return -EINVAL;
11438         }
11439
11440         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
11441                 err = dev_uc_add_excl(dev, addr);
11442         else if (is_multicast_ether_addr(addr))
11443                 err = dev_mc_add_excl(dev, addr);
11444         else
11445                 err = -EINVAL;
11446
11447         /* Only return duplicate errors if NLM_F_EXCL is set */
11448         if (err == -EEXIST && !(flags & NLM_F_EXCL))
11449                 err = 0;
11450
11451         return err;
11452 }
11453
11454 /**
11455  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
11456  * @dev: the netdev being configured
11457  * @nlh: RTNL message
11458  *
11459  * Inserts a new hardware bridge if not already created and
11460  * enables the bridging mode requested (VEB or VEPA). If the
11461  * hardware bridge has already been inserted and the request
11462  * is to change the mode then that requires a PF reset to
11463  * allow rebuild of the components with required hardware
11464  * bridge mode enabled.
11465  *
11466  * Note: expects to be called while under rtnl_lock()
11467  **/
11468 static int i40e_ndo_bridge_setlink(struct net_device *dev,
11469                                    struct nlmsghdr *nlh,
11470                                    u16 flags)
11471 {
11472         struct i40e_netdev_priv *np = netdev_priv(dev);
11473         struct i40e_vsi *vsi = np->vsi;
11474         struct i40e_pf *pf = vsi->back;
11475         struct i40e_veb *veb = NULL;
11476         struct nlattr *attr, *br_spec;
11477         int i, rem;
11478
11479         /* Only for PF VSI for now */
11480         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
11481                 return -EOPNOTSUPP;
11482
11483         /* Find the HW bridge for PF VSI */
11484         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
11485                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
11486                         veb = pf->veb[i];
11487         }
11488
11489         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
11490
11491         nla_for_each_nested(attr, br_spec, rem) {
11492                 __u16 mode;
11493
11494                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
11495                         continue;
11496
11497                 mode = nla_get_u16(attr);
11498                 if ((mode != BRIDGE_MODE_VEPA) &&
11499                     (mode != BRIDGE_MODE_VEB))
11500                         return -EINVAL;
11501
11502                 /* Insert a new HW bridge */
11503                 if (!veb) {
11504                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
11505                                              vsi->tc_config.enabled_tc);
11506                         if (veb) {
11507                                 veb->bridge_mode = mode;
11508                                 i40e_config_bridge_mode(veb);
11509                         } else {
11510                                 /* No Bridge HW offload available */
11511                                 return -ENOENT;
11512                         }
11513                         break;
11514                 } else if (mode != veb->bridge_mode) {
11515                         /* Existing HW bridge but different mode needs reset */
11516                         veb->bridge_mode = mode;
11517                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
11518                         if (mode == BRIDGE_MODE_VEB)
11519                                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
11520                         else
11521                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
11522                         i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
11523                         break;
11524                 }
11525         }
11526
11527         return 0;
11528 }
11529
11530 /**
11531  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
11532  * @skb: skb buff
11533  * @pid: process id
11534  * @seq: RTNL message seq #
11535  * @dev: the netdev being configured
11536  * @filter_mask: unused
11537  * @nlflags: netlink flags passed in
11538  *
11539  * Return the mode in which the hardware bridge is operating in
11540  * i.e VEB or VEPA.
11541  **/
11542 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
11543                                    struct net_device *dev,
11544                                    u32 __always_unused filter_mask,
11545                                    int nlflags)
11546 {
11547         struct i40e_netdev_priv *np = netdev_priv(dev);
11548         struct i40e_vsi *vsi = np->vsi;
11549         struct i40e_pf *pf = vsi->back;
11550         struct i40e_veb *veb = NULL;
11551         int i;
11552
11553         /* Only for PF VSI for now */
11554         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
11555                 return -EOPNOTSUPP;
11556
11557         /* Find the HW bridge for the PF VSI */
11558         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
11559                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
11560                         veb = pf->veb[i];
11561         }
11562
11563         if (!veb)
11564                 return 0;
11565
11566         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
11567                                        0, 0, nlflags, filter_mask, NULL);
11568 }
11569
11570 /**
11571  * i40e_features_check - Validate encapsulated packet conforms to limits
11572  * @skb: skb buff
11573  * @dev: This physical port's netdev
11574  * @features: Offload features that the stack believes apply
11575  **/
11576 static netdev_features_t i40e_features_check(struct sk_buff *skb,
11577                                              struct net_device *dev,
11578                                              netdev_features_t features)
11579 {
11580         size_t len;
11581
11582         /* No point in doing any of this if neither checksum nor GSO are
11583          * being requested for this frame.  We can rule out both by just
11584          * checking for CHECKSUM_PARTIAL
11585          */
11586         if (skb->ip_summed != CHECKSUM_PARTIAL)
11587                 return features;
11588
11589         /* We cannot support GSO if the MSS is going to be less than
11590          * 64 bytes.  If it is then we need to drop support for GSO.
11591          */
11592         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
11593                 features &= ~NETIF_F_GSO_MASK;
11594
11595         /* MACLEN can support at most 63 words */
11596         len = skb_network_header(skb) - skb->data;
11597         if (len & ~(63 * 2))
11598                 goto out_err;
11599
11600         /* IPLEN and EIPLEN can support at most 127 dwords */
11601         len = skb_transport_header(skb) - skb_network_header(skb);
11602         if (len & ~(127 * 4))
11603                 goto out_err;
11604
11605         if (skb->encapsulation) {
11606                 /* L4TUNLEN can support 127 words */
11607                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
11608                 if (len & ~(127 * 2))
11609                         goto out_err;
11610
11611                 /* IPLEN can support at most 127 dwords */
11612                 len = skb_inner_transport_header(skb) -
11613                       skb_inner_network_header(skb);
11614                 if (len & ~(127 * 4))
11615                         goto out_err;
11616         }
11617
11618         /* No need to validate L4LEN as TCP is the only protocol with a
11619          * a flexible value and we support all possible values supported
11620          * by TCP, which is at most 15 dwords
11621          */
11622
11623         return features;
11624 out_err:
11625         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
11626 }
11627
11628 /**
11629  * i40e_xdp_setup - add/remove an XDP program
11630  * @vsi: VSI to changed
11631  * @prog: XDP program
11632  **/
11633 static int i40e_xdp_setup(struct i40e_vsi *vsi,
11634                           struct bpf_prog *prog)
11635 {
11636         int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
11637         struct i40e_pf *pf = vsi->back;
11638         struct bpf_prog *old_prog;
11639         bool need_reset;
11640         int i;
11641
11642         /* Don't allow frames that span over multiple buffers */
11643         if (frame_size > vsi->rx_buf_len)
11644                 return -EINVAL;
11645
11646         if (!i40e_enabled_xdp_vsi(vsi) && !prog)
11647                 return 0;
11648
11649         /* When turning XDP on->off/off->on we reset and rebuild the rings. */
11650         need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
11651
11652         if (need_reset)
11653                 i40e_prep_for_reset(pf, true);
11654
11655         old_prog = xchg(&vsi->xdp_prog, prog);
11656
11657         if (need_reset)
11658                 i40e_reset_and_rebuild(pf, true, true);
11659
11660         for (i = 0; i < vsi->num_queue_pairs; i++)
11661                 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
11662
11663         if (old_prog)
11664                 bpf_prog_put(old_prog);
11665
11666         return 0;
11667 }
11668
11669 /**
11670  * i40e_xdp - implements ndo_bpf for i40e
11671  * @dev: netdevice
11672  * @xdp: XDP command
11673  **/
11674 static int i40e_xdp(struct net_device *dev,
11675                     struct netdev_bpf *xdp)
11676 {
11677         struct i40e_netdev_priv *np = netdev_priv(dev);
11678         struct i40e_vsi *vsi = np->vsi;
11679
11680         if (vsi->type != I40E_VSI_MAIN)
11681                 return -EINVAL;
11682
11683         switch (xdp->command) {
11684         case XDP_SETUP_PROG:
11685                 return i40e_xdp_setup(vsi, xdp->prog);
11686         case XDP_QUERY_PROG:
11687                 xdp->prog_attached = i40e_enabled_xdp_vsi(vsi);
11688                 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0;
11689                 return 0;
11690         default:
11691                 return -EINVAL;
11692         }
11693 }
11694
11695 static const struct net_device_ops i40e_netdev_ops = {
11696         .ndo_open               = i40e_open,
11697         .ndo_stop               = i40e_close,
11698         .ndo_start_xmit         = i40e_lan_xmit_frame,
11699         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
11700         .ndo_set_rx_mode        = i40e_set_rx_mode,
11701         .ndo_validate_addr      = eth_validate_addr,
11702         .ndo_set_mac_address    = i40e_set_mac,
11703         .ndo_change_mtu         = i40e_change_mtu,
11704         .ndo_do_ioctl           = i40e_ioctl,
11705         .ndo_tx_timeout         = i40e_tx_timeout,
11706         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
11707         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
11708 #ifdef CONFIG_NET_POLL_CONTROLLER
11709         .ndo_poll_controller    = i40e_netpoll,
11710 #endif
11711         .ndo_setup_tc           = __i40e_setup_tc,
11712         .ndo_set_features       = i40e_set_features,
11713         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
11714         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
11715         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
11716         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
11717         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
11718         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
11719         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
11720         .ndo_udp_tunnel_add     = i40e_udp_tunnel_add,
11721         .ndo_udp_tunnel_del     = i40e_udp_tunnel_del,
11722         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
11723         .ndo_fdb_add            = i40e_ndo_fdb_add,
11724         .ndo_features_check     = i40e_features_check,
11725         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
11726         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
11727         .ndo_bpf                = i40e_xdp,
11728 };
11729
11730 /**
11731  * i40e_config_netdev - Setup the netdev flags
11732  * @vsi: the VSI being configured
11733  *
11734  * Returns 0 on success, negative value on failure
11735  **/
11736 static int i40e_config_netdev(struct i40e_vsi *vsi)
11737 {
11738         struct i40e_pf *pf = vsi->back;
11739         struct i40e_hw *hw = &pf->hw;
11740         struct i40e_netdev_priv *np;
11741         struct net_device *netdev;
11742         u8 broadcast[ETH_ALEN];
11743         u8 mac_addr[ETH_ALEN];
11744         int etherdev_size;
11745         netdev_features_t hw_enc_features;
11746         netdev_features_t hw_features;
11747
11748         etherdev_size = sizeof(struct i40e_netdev_priv);
11749         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
11750         if (!netdev)
11751                 return -ENOMEM;
11752
11753         vsi->netdev = netdev;
11754         np = netdev_priv(netdev);
11755         np->vsi = vsi;
11756
11757         hw_enc_features = NETIF_F_SG                    |
11758                           NETIF_F_IP_CSUM               |
11759                           NETIF_F_IPV6_CSUM             |
11760                           NETIF_F_HIGHDMA               |
11761                           NETIF_F_SOFT_FEATURES         |
11762                           NETIF_F_TSO                   |
11763                           NETIF_F_TSO_ECN               |
11764                           NETIF_F_TSO6                  |
11765                           NETIF_F_GSO_GRE               |
11766                           NETIF_F_GSO_GRE_CSUM          |
11767                           NETIF_F_GSO_PARTIAL           |
11768                           NETIF_F_GSO_UDP_TUNNEL        |
11769                           NETIF_F_GSO_UDP_TUNNEL_CSUM   |
11770                           NETIF_F_SCTP_CRC              |
11771                           NETIF_F_RXHASH                |
11772                           NETIF_F_RXCSUM                |
11773                           0;
11774
11775         if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
11776                 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
11777
11778         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
11779
11780         netdev->hw_enc_features |= hw_enc_features;
11781
11782         /* record features VLANs can make use of */
11783         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
11784
11785         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
11786                 netdev->hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
11787
11788         hw_features = hw_enc_features           |
11789                       NETIF_F_HW_VLAN_CTAG_TX   |
11790                       NETIF_F_HW_VLAN_CTAG_RX;
11791
11792         netdev->hw_features |= hw_features;
11793
11794         netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
11795         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
11796
11797         if (vsi->type == I40E_VSI_MAIN) {
11798                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
11799                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
11800                 /* The following steps are necessary for two reasons. First,
11801                  * some older NVM configurations load a default MAC-VLAN
11802                  * filter that will accept any tagged packet, and we want to
11803                  * replace this with a normal filter. Additionally, it is
11804                  * possible our MAC address was provided by the platform using
11805                  * Open Firmware or similar.
11806                  *
11807                  * Thus, we need to remove the default filter and install one
11808                  * specific to the MAC address.
11809                  */
11810                 i40e_rm_default_mac_filter(vsi, mac_addr);
11811                 spin_lock_bh(&vsi->mac_filter_hash_lock);
11812                 i40e_add_mac_filter(vsi, mac_addr);
11813                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
11814         } else {
11815                 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
11816                  * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
11817                  * the end, which is 4 bytes long, so force truncation of the
11818                  * original name by IFNAMSIZ - 4
11819                  */
11820                 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
11821                          IFNAMSIZ - 4,
11822                          pf->vsi[pf->lan_vsi]->netdev->name);
11823                 random_ether_addr(mac_addr);
11824
11825                 spin_lock_bh(&vsi->mac_filter_hash_lock);
11826                 i40e_add_mac_filter(vsi, mac_addr);
11827                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
11828         }
11829
11830         /* Add the broadcast filter so that we initially will receive
11831          * broadcast packets. Note that when a new VLAN is first added the
11832          * driver will convert all filters marked I40E_VLAN_ANY into VLAN
11833          * specific filters as part of transitioning into "vlan" operation.
11834          * When more VLANs are added, the driver will copy each existing MAC
11835          * filter and add it for the new VLAN.
11836          *
11837          * Broadcast filters are handled specially by
11838          * i40e_sync_filters_subtask, as the driver must to set the broadcast
11839          * promiscuous bit instead of adding this directly as a MAC/VLAN
11840          * filter. The subtask will update the correct broadcast promiscuous
11841          * bits as VLANs become active or inactive.
11842          */
11843         eth_broadcast_addr(broadcast);
11844         spin_lock_bh(&vsi->mac_filter_hash_lock);
11845         i40e_add_mac_filter(vsi, broadcast);
11846         spin_unlock_bh(&vsi->mac_filter_hash_lock);
11847
11848         ether_addr_copy(netdev->dev_addr, mac_addr);
11849         ether_addr_copy(netdev->perm_addr, mac_addr);
11850
11851         netdev->priv_flags |= IFF_UNICAST_FLT;
11852         netdev->priv_flags |= IFF_SUPP_NOFCS;
11853         /* Setup netdev TC information */
11854         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
11855
11856         netdev->netdev_ops = &i40e_netdev_ops;
11857         netdev->watchdog_timeo = 5 * HZ;
11858         i40e_set_ethtool_ops(netdev);
11859
11860         /* MTU range: 68 - 9706 */
11861         netdev->min_mtu = ETH_MIN_MTU;
11862         netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
11863
11864         return 0;
11865 }
11866
11867 /**
11868  * i40e_vsi_delete - Delete a VSI from the switch
11869  * @vsi: the VSI being removed
11870  *
11871  * Returns 0 on success, negative value on failure
11872  **/
11873 static void i40e_vsi_delete(struct i40e_vsi *vsi)
11874 {
11875         /* remove default VSI is not allowed */
11876         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
11877                 return;
11878
11879         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
11880 }
11881
11882 /**
11883  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
11884  * @vsi: the VSI being queried
11885  *
11886  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
11887  **/
11888 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
11889 {
11890         struct i40e_veb *veb;
11891         struct i40e_pf *pf = vsi->back;
11892
11893         /* Uplink is not a bridge so default to VEB */
11894         if (vsi->veb_idx == I40E_NO_VEB)
11895                 return 1;
11896
11897         veb = pf->veb[vsi->veb_idx];
11898         if (!veb) {
11899                 dev_info(&pf->pdev->dev,
11900                          "There is no veb associated with the bridge\n");
11901                 return -ENOENT;
11902         }
11903
11904         /* Uplink is a bridge in VEPA mode */
11905         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
11906                 return 0;
11907         } else {
11908                 /* Uplink is a bridge in VEB mode */
11909                 return 1;
11910         }
11911
11912         /* VEPA is now default bridge, so return 0 */
11913         return 0;
11914 }
11915
11916 /**
11917  * i40e_add_vsi - Add a VSI to the switch
11918  * @vsi: the VSI being configured
11919  *
11920  * This initializes a VSI context depending on the VSI type to be added and
11921  * passes it down to the add_vsi aq command.
11922  **/
11923 static int i40e_add_vsi(struct i40e_vsi *vsi)
11924 {
11925         int ret = -ENODEV;
11926         struct i40e_pf *pf = vsi->back;
11927         struct i40e_hw *hw = &pf->hw;
11928         struct i40e_vsi_context ctxt;
11929         struct i40e_mac_filter *f;
11930         struct hlist_node *h;
11931         int bkt;
11932
11933         u8 enabled_tc = 0x1; /* TC0 enabled */
11934         int f_count = 0;
11935
11936         memset(&ctxt, 0, sizeof(ctxt));
11937         switch (vsi->type) {
11938         case I40E_VSI_MAIN:
11939                 /* The PF's main VSI is already setup as part of the
11940                  * device initialization, so we'll not bother with
11941                  * the add_vsi call, but we will retrieve the current
11942                  * VSI context.
11943                  */
11944                 ctxt.seid = pf->main_vsi_seid;
11945                 ctxt.pf_num = pf->hw.pf_id;
11946                 ctxt.vf_num = 0;
11947                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
11948                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
11949                 if (ret) {
11950                         dev_info(&pf->pdev->dev,
11951                                  "couldn't get PF vsi config, err %s aq_err %s\n",
11952                                  i40e_stat_str(&pf->hw, ret),
11953                                  i40e_aq_str(&pf->hw,
11954                                              pf->hw.aq.asq_last_status));
11955                         return -ENOENT;
11956                 }
11957                 vsi->info = ctxt.info;
11958                 vsi->info.valid_sections = 0;
11959
11960                 vsi->seid = ctxt.seid;
11961                 vsi->id = ctxt.vsi_number;
11962
11963                 enabled_tc = i40e_pf_get_tc_map(pf);
11964
11965                 /* Source pruning is enabled by default, so the flag is
11966                  * negative logic - if it's set, we need to fiddle with
11967                  * the VSI to disable source pruning.
11968                  */
11969                 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) {
11970                         memset(&ctxt, 0, sizeof(ctxt));
11971                         ctxt.seid = pf->main_vsi_seid;
11972                         ctxt.pf_num = pf->hw.pf_id;
11973                         ctxt.vf_num = 0;
11974                         ctxt.info.valid_sections |=
11975                                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
11976                         ctxt.info.switch_id =
11977                                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
11978                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
11979                         if (ret) {
11980                                 dev_info(&pf->pdev->dev,
11981                                          "update vsi failed, err %s aq_err %s\n",
11982                                          i40e_stat_str(&pf->hw, ret),
11983                                          i40e_aq_str(&pf->hw,
11984                                                      pf->hw.aq.asq_last_status));
11985                                 ret = -ENOENT;
11986                                 goto err;
11987                         }
11988                 }
11989
11990                 /* MFP mode setup queue map and update VSI */
11991                 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
11992                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
11993                         memset(&ctxt, 0, sizeof(ctxt));
11994                         ctxt.seid = pf->main_vsi_seid;
11995                         ctxt.pf_num = pf->hw.pf_id;
11996                         ctxt.vf_num = 0;
11997                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
11998                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
11999                         if (ret) {
12000                                 dev_info(&pf->pdev->dev,
12001                                          "update vsi failed, err %s aq_err %s\n",
12002                                          i40e_stat_str(&pf->hw, ret),
12003                                          i40e_aq_str(&pf->hw,
12004                                                     pf->hw.aq.asq_last_status));
12005                                 ret = -ENOENT;
12006                                 goto err;
12007                         }
12008                         /* update the local VSI info queue map */
12009                         i40e_vsi_update_queue_map(vsi, &ctxt);
12010                         vsi->info.valid_sections = 0;
12011                 } else {
12012                         /* Default/Main VSI is only enabled for TC0
12013                          * reconfigure it to enable all TCs that are
12014                          * available on the port in SFP mode.
12015                          * For MFP case the iSCSI PF would use this
12016                          * flow to enable LAN+iSCSI TC.
12017                          */
12018                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
12019                         if (ret) {
12020                                 /* Single TC condition is not fatal,
12021                                  * message and continue
12022                                  */
12023                                 dev_info(&pf->pdev->dev,
12024                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
12025                                          enabled_tc,
12026                                          i40e_stat_str(&pf->hw, ret),
12027                                          i40e_aq_str(&pf->hw,
12028                                                     pf->hw.aq.asq_last_status));
12029                         }
12030                 }
12031                 break;
12032
12033         case I40E_VSI_FDIR:
12034                 ctxt.pf_num = hw->pf_id;
12035                 ctxt.vf_num = 0;
12036                 ctxt.uplink_seid = vsi->uplink_seid;
12037                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
12038                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
12039                 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
12040                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
12041                         ctxt.info.valid_sections |=
12042                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
12043                         ctxt.info.switch_id =
12044                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
12045                 }
12046                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
12047                 break;
12048
12049         case I40E_VSI_VMDQ2:
12050                 ctxt.pf_num = hw->pf_id;
12051                 ctxt.vf_num = 0;
12052                 ctxt.uplink_seid = vsi->uplink_seid;
12053                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
12054                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
12055
12056                 /* This VSI is connected to VEB so the switch_id
12057                  * should be set to zero by default.
12058                  */
12059                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
12060                         ctxt.info.valid_sections |=
12061                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
12062                         ctxt.info.switch_id =
12063                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
12064                 }
12065
12066                 /* Setup the VSI tx/rx queue map for TC0 only for now */
12067                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
12068                 break;
12069
12070         case I40E_VSI_SRIOV:
12071                 ctxt.pf_num = hw->pf_id;
12072                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
12073                 ctxt.uplink_seid = vsi->uplink_seid;
12074                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
12075                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
12076
12077                 /* This VSI is connected to VEB so the switch_id
12078                  * should be set to zero by default.
12079                  */
12080                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
12081                         ctxt.info.valid_sections |=
12082                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
12083                         ctxt.info.switch_id =
12084                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
12085                 }
12086
12087                 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
12088                         ctxt.info.valid_sections |=
12089                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
12090                         ctxt.info.queueing_opt_flags |=
12091                                 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
12092                                  I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
12093                 }
12094
12095                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
12096                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
12097                 if (pf->vf[vsi->vf_id].spoofchk) {
12098                         ctxt.info.valid_sections |=
12099                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
12100                         ctxt.info.sec_flags |=
12101                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
12102                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
12103                 }
12104                 /* Setup the VSI tx/rx queue map for TC0 only for now */
12105                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
12106                 break;
12107
12108         case I40E_VSI_IWARP:
12109                 /* send down message to iWARP */
12110                 break;
12111
12112         default:
12113                 return -ENODEV;
12114         }
12115
12116         if (vsi->type != I40E_VSI_MAIN) {
12117                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
12118                 if (ret) {
12119                         dev_info(&vsi->back->pdev->dev,
12120                                  "add vsi failed, err %s aq_err %s\n",
12121                                  i40e_stat_str(&pf->hw, ret),
12122                                  i40e_aq_str(&pf->hw,
12123                                              pf->hw.aq.asq_last_status));
12124                         ret = -ENOENT;
12125                         goto err;
12126                 }
12127                 vsi->info = ctxt.info;
12128                 vsi->info.valid_sections = 0;
12129                 vsi->seid = ctxt.seid;
12130                 vsi->id = ctxt.vsi_number;
12131         }
12132
12133         vsi->active_filters = 0;
12134         clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
12135         spin_lock_bh(&vsi->mac_filter_hash_lock);
12136         /* If macvlan filters already exist, force them to get loaded */
12137         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
12138                 f->state = I40E_FILTER_NEW;
12139                 f_count++;
12140         }
12141         spin_unlock_bh(&vsi->mac_filter_hash_lock);
12142
12143         if (f_count) {
12144                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
12145                 pf->flags |= I40E_FLAG_FILTER_SYNC;
12146         }
12147
12148         /* Update VSI BW information */
12149         ret = i40e_vsi_get_bw_info(vsi);
12150         if (ret) {
12151                 dev_info(&pf->pdev->dev,
12152                          "couldn't get vsi bw info, err %s aq_err %s\n",
12153                          i40e_stat_str(&pf->hw, ret),
12154                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
12155                 /* VSI is already added so not tearing that up */
12156                 ret = 0;
12157         }
12158
12159 err:
12160         return ret;
12161 }
12162
12163 /**
12164  * i40e_vsi_release - Delete a VSI and free its resources
12165  * @vsi: the VSI being removed
12166  *
12167  * Returns 0 on success or < 0 on error
12168  **/
12169 int i40e_vsi_release(struct i40e_vsi *vsi)
12170 {
12171         struct i40e_mac_filter *f;
12172         struct hlist_node *h;
12173         struct i40e_veb *veb = NULL;
12174         struct i40e_pf *pf;
12175         u16 uplink_seid;
12176         int i, n, bkt;
12177
12178         pf = vsi->back;
12179
12180         /* release of a VEB-owner or last VSI is not allowed */
12181         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
12182                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
12183                          vsi->seid, vsi->uplink_seid);
12184                 return -ENODEV;
12185         }
12186         if (vsi == pf->vsi[pf->lan_vsi] &&
12187             !test_bit(__I40E_DOWN, pf->state)) {
12188                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
12189                 return -ENODEV;
12190         }
12191
12192         uplink_seid = vsi->uplink_seid;
12193         if (vsi->type != I40E_VSI_SRIOV) {
12194                 if (vsi->netdev_registered) {
12195                         vsi->netdev_registered = false;
12196                         if (vsi->netdev) {
12197                                 /* results in a call to i40e_close() */
12198                                 unregister_netdev(vsi->netdev);
12199                         }
12200                 } else {
12201                         i40e_vsi_close(vsi);
12202                 }
12203                 i40e_vsi_disable_irq(vsi);
12204         }
12205
12206         spin_lock_bh(&vsi->mac_filter_hash_lock);
12207
12208         /* clear the sync flag on all filters */
12209         if (vsi->netdev) {
12210                 __dev_uc_unsync(vsi->netdev, NULL);
12211                 __dev_mc_unsync(vsi->netdev, NULL);
12212         }
12213
12214         /* make sure any remaining filters are marked for deletion */
12215         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
12216                 __i40e_del_filter(vsi, f);
12217
12218         spin_unlock_bh(&vsi->mac_filter_hash_lock);
12219
12220         i40e_sync_vsi_filters(vsi);
12221
12222         i40e_vsi_delete(vsi);
12223         i40e_vsi_free_q_vectors(vsi);
12224         if (vsi->netdev) {
12225                 free_netdev(vsi->netdev);
12226                 vsi->netdev = NULL;
12227         }
12228         i40e_vsi_clear_rings(vsi);
12229         i40e_vsi_clear(vsi);
12230
12231         /* If this was the last thing on the VEB, except for the
12232          * controlling VSI, remove the VEB, which puts the controlling
12233          * VSI onto the next level down in the switch.
12234          *
12235          * Well, okay, there's one more exception here: don't remove
12236          * the orphan VEBs yet.  We'll wait for an explicit remove request
12237          * from up the network stack.
12238          */
12239         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
12240                 if (pf->vsi[i] &&
12241                     pf->vsi[i]->uplink_seid == uplink_seid &&
12242                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
12243                         n++;      /* count the VSIs */
12244                 }
12245         }
12246         for (i = 0; i < I40E_MAX_VEB; i++) {
12247                 if (!pf->veb[i])
12248                         continue;
12249                 if (pf->veb[i]->uplink_seid == uplink_seid)
12250                         n++;     /* count the VEBs */
12251                 if (pf->veb[i]->seid == uplink_seid)
12252                         veb = pf->veb[i];
12253         }
12254         if (n == 0 && veb && veb->uplink_seid != 0)
12255                 i40e_veb_release(veb);
12256
12257         return 0;
12258 }
12259
12260 /**
12261  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
12262  * @vsi: ptr to the VSI
12263  *
12264  * This should only be called after i40e_vsi_mem_alloc() which allocates the
12265  * corresponding SW VSI structure and initializes num_queue_pairs for the
12266  * newly allocated VSI.
12267  *
12268  * Returns 0 on success or negative on failure
12269  **/
12270 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
12271 {
12272         int ret = -ENOENT;
12273         struct i40e_pf *pf = vsi->back;
12274
12275         if (vsi->q_vectors[0]) {
12276                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
12277                          vsi->seid);
12278                 return -EEXIST;
12279         }
12280
12281         if (vsi->base_vector) {
12282                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
12283                          vsi->seid, vsi->base_vector);
12284                 return -EEXIST;
12285         }
12286
12287         ret = i40e_vsi_alloc_q_vectors(vsi);
12288         if (ret) {
12289                 dev_info(&pf->pdev->dev,
12290                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
12291                          vsi->num_q_vectors, vsi->seid, ret);
12292                 vsi->num_q_vectors = 0;
12293                 goto vector_setup_out;
12294         }
12295
12296         /* In Legacy mode, we do not have to get any other vector since we
12297          * piggyback on the misc/ICR0 for queue interrupts.
12298         */
12299         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
12300                 return ret;
12301         if (vsi->num_q_vectors)
12302                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
12303                                                  vsi->num_q_vectors, vsi->idx);
12304         if (vsi->base_vector < 0) {
12305                 dev_info(&pf->pdev->dev,
12306                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
12307                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
12308                 i40e_vsi_free_q_vectors(vsi);
12309                 ret = -ENOENT;
12310                 goto vector_setup_out;
12311         }
12312
12313 vector_setup_out:
12314         return ret;
12315 }
12316
12317 /**
12318  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
12319  * @vsi: pointer to the vsi.
12320  *
12321  * This re-allocates a vsi's queue resources.
12322  *
12323  * Returns pointer to the successfully allocated and configured VSI sw struct
12324  * on success, otherwise returns NULL on failure.
12325  **/
12326 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
12327 {
12328         u16 alloc_queue_pairs;
12329         struct i40e_pf *pf;
12330         u8 enabled_tc;
12331         int ret;
12332
12333         if (!vsi)
12334                 return NULL;
12335
12336         pf = vsi->back;
12337
12338         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
12339         i40e_vsi_clear_rings(vsi);
12340
12341         i40e_vsi_free_arrays(vsi, false);
12342         i40e_set_num_rings_in_vsi(vsi);
12343         ret = i40e_vsi_alloc_arrays(vsi, false);
12344         if (ret)
12345                 goto err_vsi;
12346
12347         alloc_queue_pairs = vsi->alloc_queue_pairs *
12348                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
12349
12350         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
12351         if (ret < 0) {
12352                 dev_info(&pf->pdev->dev,
12353                          "failed to get tracking for %d queues for VSI %d err %d\n",
12354                          alloc_queue_pairs, vsi->seid, ret);
12355                 goto err_vsi;
12356         }
12357         vsi->base_queue = ret;
12358
12359         /* Update the FW view of the VSI. Force a reset of TC and queue
12360          * layout configurations.
12361          */
12362         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
12363         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
12364         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
12365         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
12366         if (vsi->type == I40E_VSI_MAIN)
12367                 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
12368
12369         /* assign it some queues */
12370         ret = i40e_alloc_rings(vsi);
12371         if (ret)
12372                 goto err_rings;
12373
12374         /* map all of the rings to the q_vectors */
12375         i40e_vsi_map_rings_to_vectors(vsi);
12376         return vsi;
12377
12378 err_rings:
12379         i40e_vsi_free_q_vectors(vsi);
12380         if (vsi->netdev_registered) {
12381                 vsi->netdev_registered = false;
12382                 unregister_netdev(vsi->netdev);
12383                 free_netdev(vsi->netdev);
12384                 vsi->netdev = NULL;
12385         }
12386         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
12387 err_vsi:
12388         i40e_vsi_clear(vsi);
12389         return NULL;
12390 }
12391
12392 /**
12393  * i40e_vsi_setup - Set up a VSI by a given type
12394  * @pf: board private structure
12395  * @type: VSI type
12396  * @uplink_seid: the switch element to link to
12397  * @param1: usage depends upon VSI type. For VF types, indicates VF id
12398  *
12399  * This allocates the sw VSI structure and its queue resources, then add a VSI
12400  * to the identified VEB.
12401  *
12402  * Returns pointer to the successfully allocated and configure VSI sw struct on
12403  * success, otherwise returns NULL on failure.
12404  **/
12405 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
12406                                 u16 uplink_seid, u32 param1)
12407 {
12408         struct i40e_vsi *vsi = NULL;
12409         struct i40e_veb *veb = NULL;
12410         u16 alloc_queue_pairs;
12411         int ret, i;
12412         int v_idx;
12413
12414         /* The requested uplink_seid must be either
12415          *     - the PF's port seid
12416          *              no VEB is needed because this is the PF
12417          *              or this is a Flow Director special case VSI
12418          *     - seid of an existing VEB
12419          *     - seid of a VSI that owns an existing VEB
12420          *     - seid of a VSI that doesn't own a VEB
12421          *              a new VEB is created and the VSI becomes the owner
12422          *     - seid of the PF VSI, which is what creates the first VEB
12423          *              this is a special case of the previous
12424          *
12425          * Find which uplink_seid we were given and create a new VEB if needed
12426          */
12427         for (i = 0; i < I40E_MAX_VEB; i++) {
12428                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
12429                         veb = pf->veb[i];
12430                         break;
12431                 }
12432         }
12433
12434         if (!veb && uplink_seid != pf->mac_seid) {
12435
12436                 for (i = 0; i < pf->num_alloc_vsi; i++) {
12437                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
12438                                 vsi = pf->vsi[i];
12439                                 break;
12440                         }
12441                 }
12442                 if (!vsi) {
12443                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
12444                                  uplink_seid);
12445                         return NULL;
12446                 }
12447
12448                 if (vsi->uplink_seid == pf->mac_seid)
12449                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
12450                                              vsi->tc_config.enabled_tc);
12451                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
12452                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
12453                                              vsi->tc_config.enabled_tc);
12454                 if (veb) {
12455                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
12456                                 dev_info(&vsi->back->pdev->dev,
12457                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
12458                                 return NULL;
12459                         }
12460                         /* We come up by default in VEPA mode if SRIOV is not
12461                          * already enabled, in which case we can't force VEPA
12462                          * mode.
12463                          */
12464                         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
12465                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
12466                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
12467                         }
12468                         i40e_config_bridge_mode(veb);
12469                 }
12470                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12471                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12472                                 veb = pf->veb[i];
12473                 }
12474                 if (!veb) {
12475                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
12476                         return NULL;
12477                 }
12478
12479                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
12480                 uplink_seid = veb->seid;
12481         }
12482
12483         /* get vsi sw struct */
12484         v_idx = i40e_vsi_mem_alloc(pf, type);
12485         if (v_idx < 0)
12486                 goto err_alloc;
12487         vsi = pf->vsi[v_idx];
12488         if (!vsi)
12489                 goto err_alloc;
12490         vsi->type = type;
12491         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
12492
12493         if (type == I40E_VSI_MAIN)
12494                 pf->lan_vsi = v_idx;
12495         else if (type == I40E_VSI_SRIOV)
12496                 vsi->vf_id = param1;
12497         /* assign it some queues */
12498         alloc_queue_pairs = vsi->alloc_queue_pairs *
12499                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
12500
12501         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
12502         if (ret < 0) {
12503                 dev_info(&pf->pdev->dev,
12504                          "failed to get tracking for %d queues for VSI %d err=%d\n",
12505                          alloc_queue_pairs, vsi->seid, ret);
12506                 goto err_vsi;
12507         }
12508         vsi->base_queue = ret;
12509
12510         /* get a VSI from the hardware */
12511         vsi->uplink_seid = uplink_seid;
12512         ret = i40e_add_vsi(vsi);
12513         if (ret)
12514                 goto err_vsi;
12515
12516         switch (vsi->type) {
12517         /* setup the netdev if needed */
12518         case I40E_VSI_MAIN:
12519         case I40E_VSI_VMDQ2:
12520                 ret = i40e_config_netdev(vsi);
12521                 if (ret)
12522                         goto err_netdev;
12523                 ret = register_netdev(vsi->netdev);
12524                 if (ret)
12525                         goto err_netdev;
12526                 vsi->netdev_registered = true;
12527                 netif_carrier_off(vsi->netdev);
12528 #ifdef CONFIG_I40E_DCB
12529                 /* Setup DCB netlink interface */
12530                 i40e_dcbnl_setup(vsi);
12531 #endif /* CONFIG_I40E_DCB */
12532                 /* fall through */
12533
12534         case I40E_VSI_FDIR:
12535                 /* set up vectors and rings if needed */
12536                 ret = i40e_vsi_setup_vectors(vsi);
12537                 if (ret)
12538                         goto err_msix;
12539
12540                 ret = i40e_alloc_rings(vsi);
12541                 if (ret)
12542                         goto err_rings;
12543
12544                 /* map all of the rings to the q_vectors */
12545                 i40e_vsi_map_rings_to_vectors(vsi);
12546
12547                 i40e_vsi_reset_stats(vsi);
12548                 break;
12549
12550         default:
12551                 /* no netdev or rings for the other VSI types */
12552                 break;
12553         }
12554
12555         if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
12556             (vsi->type == I40E_VSI_VMDQ2)) {
12557                 ret = i40e_vsi_config_rss(vsi);
12558         }
12559         return vsi;
12560
12561 err_rings:
12562         i40e_vsi_free_q_vectors(vsi);
12563 err_msix:
12564         if (vsi->netdev_registered) {
12565                 vsi->netdev_registered = false;
12566                 unregister_netdev(vsi->netdev);
12567                 free_netdev(vsi->netdev);
12568                 vsi->netdev = NULL;
12569         }
12570 err_netdev:
12571         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
12572 err_vsi:
12573         i40e_vsi_clear(vsi);
12574 err_alloc:
12575         return NULL;
12576 }
12577
12578 /**
12579  * i40e_veb_get_bw_info - Query VEB BW information
12580  * @veb: the veb to query
12581  *
12582  * Query the Tx scheduler BW configuration data for given VEB
12583  **/
12584 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
12585 {
12586         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
12587         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
12588         struct i40e_pf *pf = veb->pf;
12589         struct i40e_hw *hw = &pf->hw;
12590         u32 tc_bw_max;
12591         int ret = 0;
12592         int i;
12593
12594         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
12595                                                   &bw_data, NULL);
12596         if (ret) {
12597                 dev_info(&pf->pdev->dev,
12598                          "query veb bw config failed, err %s aq_err %s\n",
12599                          i40e_stat_str(&pf->hw, ret),
12600                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
12601                 goto out;
12602         }
12603
12604         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
12605                                                    &ets_data, NULL);
12606         if (ret) {
12607                 dev_info(&pf->pdev->dev,
12608                          "query veb bw ets config failed, err %s aq_err %s\n",
12609                          i40e_stat_str(&pf->hw, ret),
12610                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
12611                 goto out;
12612         }
12613
12614         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
12615         veb->bw_max_quanta = ets_data.tc_bw_max;
12616         veb->is_abs_credits = bw_data.absolute_credits_enable;
12617         veb->enabled_tc = ets_data.tc_valid_bits;
12618         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
12619                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
12620         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
12621                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
12622                 veb->bw_tc_limit_credits[i] =
12623                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
12624                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
12625         }
12626
12627 out:
12628         return ret;
12629 }
12630
12631 /**
12632  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
12633  * @pf: board private structure
12634  *
12635  * On error: returns error code (negative)
12636  * On success: returns vsi index in PF (positive)
12637  **/
12638 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
12639 {
12640         int ret = -ENOENT;
12641         struct i40e_veb *veb;
12642         int i;
12643
12644         /* Need to protect the allocation of switch elements at the PF level */
12645         mutex_lock(&pf->switch_mutex);
12646
12647         /* VEB list may be fragmented if VEB creation/destruction has
12648          * been happening.  We can afford to do a quick scan to look
12649          * for any free slots in the list.
12650          *
12651          * find next empty veb slot, looping back around if necessary
12652          */
12653         i = 0;
12654         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
12655                 i++;
12656         if (i >= I40E_MAX_VEB) {
12657                 ret = -ENOMEM;
12658                 goto err_alloc_veb;  /* out of VEB slots! */
12659         }
12660
12661         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
12662         if (!veb) {
12663                 ret = -ENOMEM;
12664                 goto err_alloc_veb;
12665         }
12666         veb->pf = pf;
12667         veb->idx = i;
12668         veb->enabled_tc = 1;
12669
12670         pf->veb[i] = veb;
12671         ret = i;
12672 err_alloc_veb:
12673         mutex_unlock(&pf->switch_mutex);
12674         return ret;
12675 }
12676
12677 /**
12678  * i40e_switch_branch_release - Delete a branch of the switch tree
12679  * @branch: where to start deleting
12680  *
12681  * This uses recursion to find the tips of the branch to be
12682  * removed, deleting until we get back to and can delete this VEB.
12683  **/
12684 static void i40e_switch_branch_release(struct i40e_veb *branch)
12685 {
12686         struct i40e_pf *pf = branch->pf;
12687         u16 branch_seid = branch->seid;
12688         u16 veb_idx = branch->idx;
12689         int i;
12690
12691         /* release any VEBs on this VEB - RECURSION */
12692         for (i = 0; i < I40E_MAX_VEB; i++) {
12693                 if (!pf->veb[i])
12694                         continue;
12695                 if (pf->veb[i]->uplink_seid == branch->seid)
12696                         i40e_switch_branch_release(pf->veb[i]);
12697         }
12698
12699         /* Release the VSIs on this VEB, but not the owner VSI.
12700          *
12701          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
12702          *       the VEB itself, so don't use (*branch) after this loop.
12703          */
12704         for (i = 0; i < pf->num_alloc_vsi; i++) {
12705                 if (!pf->vsi[i])
12706                         continue;
12707                 if (pf->vsi[i]->uplink_seid == branch_seid &&
12708                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
12709                         i40e_vsi_release(pf->vsi[i]);
12710                 }
12711         }
12712
12713         /* There's one corner case where the VEB might not have been
12714          * removed, so double check it here and remove it if needed.
12715          * This case happens if the veb was created from the debugfs
12716          * commands and no VSIs were added to it.
12717          */
12718         if (pf->veb[veb_idx])
12719                 i40e_veb_release(pf->veb[veb_idx]);
12720 }
12721
12722 /**
12723  * i40e_veb_clear - remove veb struct
12724  * @veb: the veb to remove
12725  **/
12726 static void i40e_veb_clear(struct i40e_veb *veb)
12727 {
12728         if (!veb)
12729                 return;
12730
12731         if (veb->pf) {
12732                 struct i40e_pf *pf = veb->pf;
12733
12734                 mutex_lock(&pf->switch_mutex);
12735                 if (pf->veb[veb->idx] == veb)
12736                         pf->veb[veb->idx] = NULL;
12737                 mutex_unlock(&pf->switch_mutex);
12738         }
12739
12740         kfree(veb);
12741 }
12742
12743 /**
12744  * i40e_veb_release - Delete a VEB and free its resources
12745  * @veb: the VEB being removed
12746  **/
12747 void i40e_veb_release(struct i40e_veb *veb)
12748 {
12749         struct i40e_vsi *vsi = NULL;
12750         struct i40e_pf *pf;
12751         int i, n = 0;
12752
12753         pf = veb->pf;
12754
12755         /* find the remaining VSI and check for extras */
12756         for (i = 0; i < pf->num_alloc_vsi; i++) {
12757                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
12758                         n++;
12759                         vsi = pf->vsi[i];
12760                 }
12761         }
12762         if (n != 1) {
12763                 dev_info(&pf->pdev->dev,
12764                          "can't remove VEB %d with %d VSIs left\n",
12765                          veb->seid, n);
12766                 return;
12767         }
12768
12769         /* move the remaining VSI to uplink veb */
12770         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
12771         if (veb->uplink_seid) {
12772                 vsi->uplink_seid = veb->uplink_seid;
12773                 if (veb->uplink_seid == pf->mac_seid)
12774                         vsi->veb_idx = I40E_NO_VEB;
12775                 else
12776                         vsi->veb_idx = veb->veb_idx;
12777         } else {
12778                 /* floating VEB */
12779                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
12780                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
12781         }
12782
12783         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
12784         i40e_veb_clear(veb);
12785 }
12786
12787 /**
12788  * i40e_add_veb - create the VEB in the switch
12789  * @veb: the VEB to be instantiated
12790  * @vsi: the controlling VSI
12791  **/
12792 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
12793 {
12794         struct i40e_pf *pf = veb->pf;
12795         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
12796         int ret;
12797
12798         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
12799                               veb->enabled_tc, false,
12800                               &veb->seid, enable_stats, NULL);
12801
12802         /* get a VEB from the hardware */
12803         if (ret) {
12804                 dev_info(&pf->pdev->dev,
12805                          "couldn't add VEB, err %s aq_err %s\n",
12806                          i40e_stat_str(&pf->hw, ret),
12807                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
12808                 return -EPERM;
12809         }
12810
12811         /* get statistics counter */
12812         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
12813                                          &veb->stats_idx, NULL, NULL, NULL);
12814         if (ret) {
12815                 dev_info(&pf->pdev->dev,
12816                          "couldn't get VEB statistics idx, err %s aq_err %s\n",
12817                          i40e_stat_str(&pf->hw, ret),
12818                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
12819                 return -EPERM;
12820         }
12821         ret = i40e_veb_get_bw_info(veb);
12822         if (ret) {
12823                 dev_info(&pf->pdev->dev,
12824                          "couldn't get VEB bw info, err %s aq_err %s\n",
12825                          i40e_stat_str(&pf->hw, ret),
12826                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
12827                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
12828                 return -ENOENT;
12829         }
12830
12831         vsi->uplink_seid = veb->seid;
12832         vsi->veb_idx = veb->idx;
12833         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
12834
12835         return 0;
12836 }
12837
12838 /**
12839  * i40e_veb_setup - Set up a VEB
12840  * @pf: board private structure
12841  * @flags: VEB setup flags
12842  * @uplink_seid: the switch element to link to
12843  * @vsi_seid: the initial VSI seid
12844  * @enabled_tc: Enabled TC bit-map
12845  *
12846  * This allocates the sw VEB structure and links it into the switch
12847  * It is possible and legal for this to be a duplicate of an already
12848  * existing VEB.  It is also possible for both uplink and vsi seids
12849  * to be zero, in order to create a floating VEB.
12850  *
12851  * Returns pointer to the successfully allocated VEB sw struct on
12852  * success, otherwise returns NULL on failure.
12853  **/
12854 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
12855                                 u16 uplink_seid, u16 vsi_seid,
12856                                 u8 enabled_tc)
12857 {
12858         struct i40e_veb *veb, *uplink_veb = NULL;
12859         int vsi_idx, veb_idx;
12860         int ret;
12861
12862         /* if one seid is 0, the other must be 0 to create a floating relay */
12863         if ((uplink_seid == 0 || vsi_seid == 0) &&
12864             (uplink_seid + vsi_seid != 0)) {
12865                 dev_info(&pf->pdev->dev,
12866                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
12867                          uplink_seid, vsi_seid);
12868                 return NULL;
12869         }
12870
12871         /* make sure there is such a vsi and uplink */
12872         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
12873                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
12874                         break;
12875         if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) {
12876                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
12877                          vsi_seid);
12878                 return NULL;
12879         }
12880
12881         if (uplink_seid && uplink_seid != pf->mac_seid) {
12882                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
12883                         if (pf->veb[veb_idx] &&
12884                             pf->veb[veb_idx]->seid == uplink_seid) {
12885                                 uplink_veb = pf->veb[veb_idx];
12886                                 break;
12887                         }
12888                 }
12889                 if (!uplink_veb) {
12890                         dev_info(&pf->pdev->dev,
12891                                  "uplink seid %d not found\n", uplink_seid);
12892                         return NULL;
12893                 }
12894         }
12895
12896         /* get veb sw struct */
12897         veb_idx = i40e_veb_mem_alloc(pf);
12898         if (veb_idx < 0)
12899                 goto err_alloc;
12900         veb = pf->veb[veb_idx];
12901         veb->flags = flags;
12902         veb->uplink_seid = uplink_seid;
12903         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
12904         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
12905
12906         /* create the VEB in the switch */
12907         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
12908         if (ret)
12909                 goto err_veb;
12910         if (vsi_idx == pf->lan_vsi)
12911                 pf->lan_veb = veb->idx;
12912
12913         return veb;
12914
12915 err_veb:
12916         i40e_veb_clear(veb);
12917 err_alloc:
12918         return NULL;
12919 }
12920
12921 /**
12922  * i40e_setup_pf_switch_element - set PF vars based on switch type
12923  * @pf: board private structure
12924  * @ele: element we are building info from
12925  * @num_reported: total number of elements
12926  * @printconfig: should we print the contents
12927  *
12928  * helper function to assist in extracting a few useful SEID values.
12929  **/
12930 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
12931                                 struct i40e_aqc_switch_config_element_resp *ele,
12932                                 u16 num_reported, bool printconfig)
12933 {
12934         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
12935         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
12936         u8 element_type = ele->element_type;
12937         u16 seid = le16_to_cpu(ele->seid);
12938
12939         if (printconfig)
12940                 dev_info(&pf->pdev->dev,
12941                          "type=%d seid=%d uplink=%d downlink=%d\n",
12942                          element_type, seid, uplink_seid, downlink_seid);
12943
12944         switch (element_type) {
12945         case I40E_SWITCH_ELEMENT_TYPE_MAC:
12946                 pf->mac_seid = seid;
12947                 break;
12948         case I40E_SWITCH_ELEMENT_TYPE_VEB:
12949                 /* Main VEB? */
12950                 if (uplink_seid != pf->mac_seid)
12951                         break;
12952                 if (pf->lan_veb == I40E_NO_VEB) {
12953                         int v;
12954
12955                         /* find existing or else empty VEB */
12956                         for (v = 0; v < I40E_MAX_VEB; v++) {
12957                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
12958                                         pf->lan_veb = v;
12959                                         break;
12960                                 }
12961                         }
12962                         if (pf->lan_veb == I40E_NO_VEB) {
12963                                 v = i40e_veb_mem_alloc(pf);
12964                                 if (v < 0)
12965                                         break;
12966                                 pf->lan_veb = v;
12967                         }
12968                 }
12969
12970                 pf->veb[pf->lan_veb]->seid = seid;
12971                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
12972                 pf->veb[pf->lan_veb]->pf = pf;
12973                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
12974                 break;
12975         case I40E_SWITCH_ELEMENT_TYPE_VSI:
12976                 if (num_reported != 1)
12977                         break;
12978                 /* This is immediately after a reset so we can assume this is
12979                  * the PF's VSI
12980                  */
12981                 pf->mac_seid = uplink_seid;
12982                 pf->pf_seid = downlink_seid;
12983                 pf->main_vsi_seid = seid;
12984                 if (printconfig)
12985                         dev_info(&pf->pdev->dev,
12986                                  "pf_seid=%d main_vsi_seid=%d\n",
12987                                  pf->pf_seid, pf->main_vsi_seid);
12988                 break;
12989         case I40E_SWITCH_ELEMENT_TYPE_PF:
12990         case I40E_SWITCH_ELEMENT_TYPE_VF:
12991         case I40E_SWITCH_ELEMENT_TYPE_EMP:
12992         case I40E_SWITCH_ELEMENT_TYPE_BMC:
12993         case I40E_SWITCH_ELEMENT_TYPE_PE:
12994         case I40E_SWITCH_ELEMENT_TYPE_PA:
12995                 /* ignore these for now */
12996                 break;
12997         default:
12998                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
12999                          element_type, seid);
13000                 break;
13001         }
13002 }
13003
13004 /**
13005  * i40e_fetch_switch_configuration - Get switch config from firmware
13006  * @pf: board private structure
13007  * @printconfig: should we print the contents
13008  *
13009  * Get the current switch configuration from the device and
13010  * extract a few useful SEID values.
13011  **/
13012 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
13013 {
13014         struct i40e_aqc_get_switch_config_resp *sw_config;
13015         u16 next_seid = 0;
13016         int ret = 0;
13017         u8 *aq_buf;
13018         int i;
13019
13020         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
13021         if (!aq_buf)
13022                 return -ENOMEM;
13023
13024         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
13025         do {
13026                 u16 num_reported, num_total;
13027
13028                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
13029                                                 I40E_AQ_LARGE_BUF,
13030                                                 &next_seid, NULL);
13031                 if (ret) {
13032                         dev_info(&pf->pdev->dev,
13033                                  "get switch config failed err %s aq_err %s\n",
13034                                  i40e_stat_str(&pf->hw, ret),
13035                                  i40e_aq_str(&pf->hw,
13036                                              pf->hw.aq.asq_last_status));
13037                         kfree(aq_buf);
13038                         return -ENOENT;
13039                 }
13040
13041                 num_reported = le16_to_cpu(sw_config->header.num_reported);
13042                 num_total = le16_to_cpu(sw_config->header.num_total);
13043
13044                 if (printconfig)
13045                         dev_info(&pf->pdev->dev,
13046                                  "header: %d reported %d total\n",
13047                                  num_reported, num_total);
13048
13049                 for (i = 0; i < num_reported; i++) {
13050                         struct i40e_aqc_switch_config_element_resp *ele =
13051                                 &sw_config->element[i];
13052
13053                         i40e_setup_pf_switch_element(pf, ele, num_reported,
13054                                                      printconfig);
13055                 }
13056         } while (next_seid != 0);
13057
13058         kfree(aq_buf);
13059         return ret;
13060 }
13061
13062 /**
13063  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
13064  * @pf: board private structure
13065  * @reinit: if the Main VSI needs to re-initialized.
13066  *
13067  * Returns 0 on success, negative value on failure
13068  **/
13069 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
13070 {
13071         u16 flags = 0;
13072         int ret;
13073
13074         /* find out what's out there already */
13075         ret = i40e_fetch_switch_configuration(pf, false);
13076         if (ret) {
13077                 dev_info(&pf->pdev->dev,
13078                          "couldn't fetch switch config, err %s aq_err %s\n",
13079                          i40e_stat_str(&pf->hw, ret),
13080                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13081                 return ret;
13082         }
13083         i40e_pf_reset_stats(pf);
13084
13085         /* set the switch config bit for the whole device to
13086          * support limited promisc or true promisc
13087          * when user requests promisc. The default is limited
13088          * promisc.
13089         */
13090
13091         if ((pf->hw.pf_id == 0) &&
13092             !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) {
13093                 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
13094                 pf->last_sw_conf_flags = flags;
13095         }
13096
13097         if (pf->hw.pf_id == 0) {
13098                 u16 valid_flags;
13099
13100                 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
13101                 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
13102                                                 NULL);
13103                 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
13104                         dev_info(&pf->pdev->dev,
13105                                  "couldn't set switch config bits, err %s aq_err %s\n",
13106                                  i40e_stat_str(&pf->hw, ret),
13107                                  i40e_aq_str(&pf->hw,
13108                                              pf->hw.aq.asq_last_status));
13109                         /* not a fatal problem, just keep going */
13110                 }
13111                 pf->last_sw_conf_valid_flags = valid_flags;
13112         }
13113
13114         /* first time setup */
13115         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
13116                 struct i40e_vsi *vsi = NULL;
13117                 u16 uplink_seid;
13118
13119                 /* Set up the PF VSI associated with the PF's main VSI
13120                  * that is already in the HW switch
13121                  */
13122                 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
13123                         uplink_seid = pf->veb[pf->lan_veb]->seid;
13124                 else
13125                         uplink_seid = pf->mac_seid;
13126                 if (pf->lan_vsi == I40E_NO_VSI)
13127                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
13128                 else if (reinit)
13129                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
13130                 if (!vsi) {
13131                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
13132                         i40e_cloud_filter_exit(pf);
13133                         i40e_fdir_teardown(pf);
13134                         return -EAGAIN;
13135                 }
13136         } else {
13137                 /* force a reset of TC and queue layout configurations */
13138                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
13139
13140                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
13141                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
13142                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
13143         }
13144         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
13145
13146         i40e_fdir_sb_setup(pf);
13147
13148         /* Setup static PF queue filter control settings */
13149         ret = i40e_setup_pf_filter_control(pf);
13150         if (ret) {
13151                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
13152                          ret);
13153                 /* Failure here should not stop continuing other steps */
13154         }
13155
13156         /* enable RSS in the HW, even for only one queue, as the stack can use
13157          * the hash
13158          */
13159         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
13160                 i40e_pf_config_rss(pf);
13161
13162         /* fill in link information and enable LSE reporting */
13163         i40e_link_event(pf);
13164
13165         /* Initialize user-specific link properties */
13166         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
13167                                   I40E_AQ_AN_COMPLETED) ? true : false);
13168
13169         i40e_ptp_init(pf);
13170
13171         /* repopulate tunnel port filters */
13172         i40e_sync_udp_filters(pf);
13173
13174         return ret;
13175 }
13176
13177 /**
13178  * i40e_determine_queue_usage - Work out queue distribution
13179  * @pf: board private structure
13180  **/
13181 static void i40e_determine_queue_usage(struct i40e_pf *pf)
13182 {
13183         int queues_left;
13184         int q_max;
13185
13186         pf->num_lan_qps = 0;
13187
13188         /* Find the max queues to be put into basic use.  We'll always be
13189          * using TC0, whether or not DCB is running, and TC0 will get the
13190          * big RSS set.
13191          */
13192         queues_left = pf->hw.func_caps.num_tx_qp;
13193
13194         if ((queues_left == 1) ||
13195             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
13196                 /* one qp for PF, no queues for anything else */
13197                 queues_left = 0;
13198                 pf->alloc_rss_size = pf->num_lan_qps = 1;
13199
13200                 /* make sure all the fancies are disabled */
13201                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
13202                                I40E_FLAG_IWARP_ENABLED  |
13203                                I40E_FLAG_FD_SB_ENABLED  |
13204                                I40E_FLAG_FD_ATR_ENABLED |
13205                                I40E_FLAG_DCB_CAPABLE    |
13206                                I40E_FLAG_DCB_ENABLED    |
13207                                I40E_FLAG_SRIOV_ENABLED  |
13208                                I40E_FLAG_VMDQ_ENABLED);
13209                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
13210         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
13211                                   I40E_FLAG_FD_SB_ENABLED |
13212                                   I40E_FLAG_FD_ATR_ENABLED |
13213                                   I40E_FLAG_DCB_CAPABLE))) {
13214                 /* one qp for PF */
13215                 pf->alloc_rss_size = pf->num_lan_qps = 1;
13216                 queues_left -= pf->num_lan_qps;
13217
13218                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
13219                                I40E_FLAG_IWARP_ENABLED  |
13220                                I40E_FLAG_FD_SB_ENABLED  |
13221                                I40E_FLAG_FD_ATR_ENABLED |
13222                                I40E_FLAG_DCB_ENABLED    |
13223                                I40E_FLAG_VMDQ_ENABLED);
13224                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
13225         } else {
13226                 /* Not enough queues for all TCs */
13227                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
13228                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
13229                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
13230                                         I40E_FLAG_DCB_ENABLED);
13231                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
13232                 }
13233
13234                 /* limit lan qps to the smaller of qps, cpus or msix */
13235                 q_max = max_t(int, pf->rss_size_max, num_online_cpus());
13236                 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
13237                 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
13238                 pf->num_lan_qps = q_max;
13239
13240                 queues_left -= pf->num_lan_qps;
13241         }
13242
13243         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
13244                 if (queues_left > 1) {
13245                         queues_left -= 1; /* save 1 queue for FD */
13246                 } else {
13247                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
13248                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
13249                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
13250                 }
13251         }
13252
13253         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
13254             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
13255                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
13256                                         (queues_left / pf->num_vf_qps));
13257                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
13258         }
13259
13260         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
13261             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
13262                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
13263                                           (queues_left / pf->num_vmdq_qps));
13264                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
13265         }
13266
13267         pf->queues_left = queues_left;
13268         dev_dbg(&pf->pdev->dev,
13269                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
13270                 pf->hw.func_caps.num_tx_qp,
13271                 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
13272                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
13273                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
13274                 queues_left);
13275 }
13276
13277 /**
13278  * i40e_setup_pf_filter_control - Setup PF static filter control
13279  * @pf: PF to be setup
13280  *
13281  * i40e_setup_pf_filter_control sets up a PF's initial filter control
13282  * settings. If PE/FCoE are enabled then it will also set the per PF
13283  * based filter sizes required for them. It also enables Flow director,
13284  * ethertype and macvlan type filter settings for the pf.
13285  *
13286  * Returns 0 on success, negative on failure
13287  **/
13288 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
13289 {
13290         struct i40e_filter_control_settings *settings = &pf->filter_settings;
13291
13292         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
13293
13294         /* Flow Director is enabled */
13295         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
13296                 settings->enable_fdir = true;
13297
13298         /* Ethtype and MACVLAN filters enabled for PF */
13299         settings->enable_ethtype = true;
13300         settings->enable_macvlan = true;
13301
13302         if (i40e_set_filter_control(&pf->hw, settings))
13303                 return -ENOENT;
13304
13305         return 0;
13306 }
13307
13308 #define INFO_STRING_LEN 255
13309 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
13310 static void i40e_print_features(struct i40e_pf *pf)
13311 {
13312         struct i40e_hw *hw = &pf->hw;
13313         char *buf;
13314         int i;
13315
13316         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
13317         if (!buf)
13318                 return;
13319
13320         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
13321 #ifdef CONFIG_PCI_IOV
13322         i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
13323 #endif
13324         i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
13325                       pf->hw.func_caps.num_vsis,
13326                       pf->vsi[pf->lan_vsi]->num_queue_pairs);
13327         if (pf->flags & I40E_FLAG_RSS_ENABLED)
13328                 i += snprintf(&buf[i], REMAIN(i), " RSS");
13329         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
13330                 i += snprintf(&buf[i], REMAIN(i), " FD_ATR");
13331         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
13332                 i += snprintf(&buf[i], REMAIN(i), " FD_SB");
13333                 i += snprintf(&buf[i], REMAIN(i), " NTUPLE");
13334         }
13335         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
13336                 i += snprintf(&buf[i], REMAIN(i), " DCB");
13337         i += snprintf(&buf[i], REMAIN(i), " VxLAN");
13338         i += snprintf(&buf[i], REMAIN(i), " Geneve");
13339         if (pf->flags & I40E_FLAG_PTP)
13340                 i += snprintf(&buf[i], REMAIN(i), " PTP");
13341         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
13342                 i += snprintf(&buf[i], REMAIN(i), " VEB");
13343         else
13344                 i += snprintf(&buf[i], REMAIN(i), " VEPA");
13345
13346         dev_info(&pf->pdev->dev, "%s\n", buf);
13347         kfree(buf);
13348         WARN_ON(i > INFO_STRING_LEN);
13349 }
13350
13351 /**
13352  * i40e_get_platform_mac_addr - get platform-specific MAC address
13353  * @pdev: PCI device information struct
13354  * @pf: board private structure
13355  *
13356  * Look up the MAC address for the device. First we'll try
13357  * eth_platform_get_mac_address, which will check Open Firmware, or arch
13358  * specific fallback. Otherwise, we'll default to the stored value in
13359  * firmware.
13360  **/
13361 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
13362 {
13363         if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
13364                 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
13365 }
13366
13367 /**
13368  * i40e_probe - Device initialization routine
13369  * @pdev: PCI device information struct
13370  * @ent: entry in i40e_pci_tbl
13371  *
13372  * i40e_probe initializes a PF identified by a pci_dev structure.
13373  * The OS initialization, configuring of the PF private structure,
13374  * and a hardware reset occur.
13375  *
13376  * Returns 0 on success, negative on failure
13377  **/
13378 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
13379 {
13380         struct i40e_aq_get_phy_abilities_resp abilities;
13381         struct i40e_pf *pf;
13382         struct i40e_hw *hw;
13383         static u16 pfs_found;
13384         u16 wol_nvm_bits;
13385         u16 link_status;
13386         int err;
13387         u32 val;
13388         u32 i;
13389         u8 set_fc_aq_fail;
13390
13391         err = pci_enable_device_mem(pdev);
13392         if (err)
13393                 return err;
13394
13395         /* set up for high or low dma */
13396         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
13397         if (err) {
13398                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
13399                 if (err) {
13400                         dev_err(&pdev->dev,
13401                                 "DMA configuration failed: 0x%x\n", err);
13402                         goto err_dma;
13403                 }
13404         }
13405
13406         /* set up pci connections */
13407         err = pci_request_mem_regions(pdev, i40e_driver_name);
13408         if (err) {
13409                 dev_info(&pdev->dev,
13410                          "pci_request_selected_regions failed %d\n", err);
13411                 goto err_pci_reg;
13412         }
13413
13414         pci_enable_pcie_error_reporting(pdev);
13415         pci_set_master(pdev);
13416
13417         /* Now that we have a PCI connection, we need to do the
13418          * low level device setup.  This is primarily setting up
13419          * the Admin Queue structures and then querying for the
13420          * device's current profile information.
13421          */
13422         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
13423         if (!pf) {
13424                 err = -ENOMEM;
13425                 goto err_pf_alloc;
13426         }
13427         pf->next_vsi = 0;
13428         pf->pdev = pdev;
13429         set_bit(__I40E_DOWN, pf->state);
13430
13431         hw = &pf->hw;
13432         hw->back = pf;
13433
13434         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
13435                                 I40E_MAX_CSR_SPACE);
13436
13437         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
13438         if (!hw->hw_addr) {
13439                 err = -EIO;
13440                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
13441                          (unsigned int)pci_resource_start(pdev, 0),
13442                          pf->ioremap_len, err);
13443                 goto err_ioremap;
13444         }
13445         hw->vendor_id = pdev->vendor;
13446         hw->device_id = pdev->device;
13447         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
13448         hw->subsystem_vendor_id = pdev->subsystem_vendor;
13449         hw->subsystem_device_id = pdev->subsystem_device;
13450         hw->bus.device = PCI_SLOT(pdev->devfn);
13451         hw->bus.func = PCI_FUNC(pdev->devfn);
13452         hw->bus.bus_id = pdev->bus->number;
13453         pf->instance = pfs_found;
13454
13455         /* Select something other than the 802.1ad ethertype for the
13456          * switch to use internally and drop on ingress.
13457          */
13458         hw->switch_tag = 0xffff;
13459         hw->first_tag = ETH_P_8021AD;
13460         hw->second_tag = ETH_P_8021Q;
13461
13462         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
13463         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
13464
13465         /* set up the locks for the AQ, do this only once in probe
13466          * and destroy them only once in remove
13467          */
13468         mutex_init(&hw->aq.asq_mutex);
13469         mutex_init(&hw->aq.arq_mutex);
13470
13471         pf->msg_enable = netif_msg_init(debug,
13472                                         NETIF_MSG_DRV |
13473                                         NETIF_MSG_PROBE |
13474                                         NETIF_MSG_LINK);
13475         if (debug < -1)
13476                 pf->hw.debug_mask = debug;
13477
13478         /* do a special CORER for clearing PXE mode once at init */
13479         if (hw->revision_id == 0 &&
13480             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
13481                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
13482                 i40e_flush(hw);
13483                 msleep(200);
13484                 pf->corer_count++;
13485
13486                 i40e_clear_pxe_mode(hw);
13487         }
13488
13489         /* Reset here to make sure all is clean and to define PF 'n' */
13490         i40e_clear_hw(hw);
13491         err = i40e_pf_reset(hw);
13492         if (err) {
13493                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
13494                 goto err_pf_reset;
13495         }
13496         pf->pfr_count++;
13497
13498         hw->aq.num_arq_entries = I40E_AQ_LEN;
13499         hw->aq.num_asq_entries = I40E_AQ_LEN;
13500         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
13501         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
13502         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
13503
13504         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
13505                  "%s-%s:misc",
13506                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
13507
13508         err = i40e_init_shared_code(hw);
13509         if (err) {
13510                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
13511                          err);
13512                 goto err_pf_reset;
13513         }
13514
13515         /* set up a default setting for link flow control */
13516         pf->hw.fc.requested_mode = I40E_FC_NONE;
13517
13518         err = i40e_init_adminq(hw);
13519         if (err) {
13520                 if (err == I40E_ERR_FIRMWARE_API_VERSION)
13521                         dev_info(&pdev->dev,
13522                                  "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
13523                 else
13524                         dev_info(&pdev->dev,
13525                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
13526
13527                 goto err_pf_reset;
13528         }
13529         i40e_get_oem_version(hw);
13530
13531         /* provide nvm, fw, api versions */
13532         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n",
13533                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
13534                  hw->aq.api_maj_ver, hw->aq.api_min_ver,
13535                  i40e_nvm_version_str(hw));
13536
13537         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
13538             hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw))
13539                 dev_info(&pdev->dev,
13540                          "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
13541         else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4)
13542                 dev_info(&pdev->dev,
13543                          "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
13544
13545         i40e_verify_eeprom(pf);
13546
13547         /* Rev 0 hardware was never productized */
13548         if (hw->revision_id < 1)
13549                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
13550
13551         i40e_clear_pxe_mode(hw);
13552         err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
13553         if (err)
13554                 goto err_adminq_setup;
13555
13556         err = i40e_sw_init(pf);
13557         if (err) {
13558                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
13559                 goto err_sw_init;
13560         }
13561
13562         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
13563                                 hw->func_caps.num_rx_qp, 0, 0);
13564         if (err) {
13565                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
13566                 goto err_init_lan_hmc;
13567         }
13568
13569         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
13570         if (err) {
13571                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
13572                 err = -ENOENT;
13573                 goto err_configure_lan_hmc;
13574         }
13575
13576         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
13577          * Ignore error return codes because if it was already disabled via
13578          * hardware settings this will fail
13579          */
13580         if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
13581                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
13582                 i40e_aq_stop_lldp(hw, true, NULL);
13583         }
13584
13585         /* allow a platform config to override the HW addr */
13586         i40e_get_platform_mac_addr(pdev, pf);
13587
13588         if (!is_valid_ether_addr(hw->mac.addr)) {
13589                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
13590                 err = -EIO;
13591                 goto err_mac_addr;
13592         }
13593         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
13594         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
13595         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
13596         if (is_valid_ether_addr(hw->mac.port_addr))
13597                 pf->hw_features |= I40E_HW_PORT_ID_VALID;
13598
13599         pci_set_drvdata(pdev, pf);
13600         pci_save_state(pdev);
13601 #ifdef CONFIG_I40E_DCB
13602         err = i40e_init_pf_dcb(pf);
13603         if (err) {
13604                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
13605                 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
13606                 /* Continue without DCB enabled */
13607         }
13608 #endif /* CONFIG_I40E_DCB */
13609
13610         /* set up periodic task facility */
13611         timer_setup(&pf->service_timer, i40e_service_timer, 0);
13612         pf->service_timer_period = HZ;
13613
13614         INIT_WORK(&pf->service_task, i40e_service_task);
13615         clear_bit(__I40E_SERVICE_SCHED, pf->state);
13616
13617         /* NVM bit on means WoL disabled for the port */
13618         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
13619         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
13620                 pf->wol_en = false;
13621         else
13622                 pf->wol_en = true;
13623         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
13624
13625         /* set up the main switch operations */
13626         i40e_determine_queue_usage(pf);
13627         err = i40e_init_interrupt_scheme(pf);
13628         if (err)
13629                 goto err_switch_setup;
13630
13631         /* The number of VSIs reported by the FW is the minimum guaranteed
13632          * to us; HW supports far more and we share the remaining pool with
13633          * the other PFs. We allocate space for more than the guarantee with
13634          * the understanding that we might not get them all later.
13635          */
13636         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
13637                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
13638         else
13639                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
13640
13641         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
13642         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
13643                           GFP_KERNEL);
13644         if (!pf->vsi) {
13645                 err = -ENOMEM;
13646                 goto err_switch_setup;
13647         }
13648
13649 #ifdef CONFIG_PCI_IOV
13650         /* prep for VF support */
13651         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
13652             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
13653             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
13654                 if (pci_num_vf(pdev))
13655                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
13656         }
13657 #endif
13658         err = i40e_setup_pf_switch(pf, false);
13659         if (err) {
13660                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
13661                 goto err_vsis;
13662         }
13663         INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list);
13664
13665         /* Make sure flow control is set according to current settings */
13666         err = i40e_set_fc(hw, &set_fc_aq_fail, true);
13667         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET)
13668                 dev_dbg(&pf->pdev->dev,
13669                         "Set fc with err %s aq_err %s on get_phy_cap\n",
13670                         i40e_stat_str(hw, err),
13671                         i40e_aq_str(hw, hw->aq.asq_last_status));
13672         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET)
13673                 dev_dbg(&pf->pdev->dev,
13674                         "Set fc with err %s aq_err %s on set_phy_config\n",
13675                         i40e_stat_str(hw, err),
13676                         i40e_aq_str(hw, hw->aq.asq_last_status));
13677         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE)
13678                 dev_dbg(&pf->pdev->dev,
13679                         "Set fc with err %s aq_err %s on get_link_info\n",
13680                         i40e_stat_str(hw, err),
13681                         i40e_aq_str(hw, hw->aq.asq_last_status));
13682
13683         /* if FDIR VSI was set up, start it now */
13684         for (i = 0; i < pf->num_alloc_vsi; i++) {
13685                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
13686                         i40e_vsi_open(pf->vsi[i]);
13687                         break;
13688                 }
13689         }
13690
13691         /* The driver only wants link up/down and module qualification
13692          * reports from firmware.  Note the negative logic.
13693          */
13694         err = i40e_aq_set_phy_int_mask(&pf->hw,
13695                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
13696                                          I40E_AQ_EVENT_MEDIA_NA |
13697                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
13698         if (err)
13699                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
13700                          i40e_stat_str(&pf->hw, err),
13701                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13702
13703         /* Reconfigure hardware for allowing smaller MSS in the case
13704          * of TSO, so that we avoid the MDD being fired and causing
13705          * a reset in the case of small MSS+TSO.
13706          */
13707         val = rd32(hw, I40E_REG_MSS);
13708         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
13709                 val &= ~I40E_REG_MSS_MIN_MASK;
13710                 val |= I40E_64BYTE_MSS;
13711                 wr32(hw, I40E_REG_MSS, val);
13712         }
13713
13714         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
13715                 msleep(75);
13716                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
13717                 if (err)
13718                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
13719                                  i40e_stat_str(&pf->hw, err),
13720                                  i40e_aq_str(&pf->hw,
13721                                              pf->hw.aq.asq_last_status));
13722         }
13723         /* The main driver is (mostly) up and happy. We need to set this state
13724          * before setting up the misc vector or we get a race and the vector
13725          * ends up disabled forever.
13726          */
13727         clear_bit(__I40E_DOWN, pf->state);
13728
13729         /* In case of MSIX we are going to setup the misc vector right here
13730          * to handle admin queue events etc. In case of legacy and MSI
13731          * the misc functionality and queue processing is combined in
13732          * the same vector and that gets setup at open.
13733          */
13734         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
13735                 err = i40e_setup_misc_vector(pf);
13736                 if (err) {
13737                         dev_info(&pdev->dev,
13738                                  "setup of misc vector failed: %d\n", err);
13739                         goto err_vsis;
13740                 }
13741         }
13742
13743 #ifdef CONFIG_PCI_IOV
13744         /* prep for VF support */
13745         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
13746             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
13747             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
13748                 /* disable link interrupts for VFs */
13749                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
13750                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
13751                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
13752                 i40e_flush(hw);
13753
13754                 if (pci_num_vf(pdev)) {
13755                         dev_info(&pdev->dev,
13756                                  "Active VFs found, allocating resources.\n");
13757                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
13758                         if (err)
13759                                 dev_info(&pdev->dev,
13760                                          "Error %d allocating resources for existing VFs\n",
13761                                          err);
13762                 }
13763         }
13764 #endif /* CONFIG_PCI_IOV */
13765
13766         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
13767                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
13768                                                       pf->num_iwarp_msix,
13769                                                       I40E_IWARP_IRQ_PILE_ID);
13770                 if (pf->iwarp_base_vector < 0) {
13771                         dev_info(&pdev->dev,
13772                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
13773                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
13774                         pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
13775                 }
13776         }
13777
13778         i40e_dbg_pf_init(pf);
13779
13780         /* tell the firmware that we're starting */
13781         i40e_send_version(pf);
13782
13783         /* since everything's happy, start the service_task timer */
13784         mod_timer(&pf->service_timer,
13785                   round_jiffies(jiffies + pf->service_timer_period));
13786
13787         /* add this PF to client device list and launch a client service task */
13788         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
13789                 err = i40e_lan_add_device(pf);
13790                 if (err)
13791                         dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
13792                                  err);
13793         }
13794
13795 #define PCI_SPEED_SIZE 8
13796 #define PCI_WIDTH_SIZE 8
13797         /* Devices on the IOSF bus do not have this information
13798          * and will report PCI Gen 1 x 1 by default so don't bother
13799          * checking them.
13800          */
13801         if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
13802                 char speed[PCI_SPEED_SIZE] = "Unknown";
13803                 char width[PCI_WIDTH_SIZE] = "Unknown";
13804
13805                 /* Get the negotiated link width and speed from PCI config
13806                  * space
13807                  */
13808                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
13809                                           &link_status);
13810
13811                 i40e_set_pci_config_data(hw, link_status);
13812
13813                 switch (hw->bus.speed) {
13814                 case i40e_bus_speed_8000:
13815                         strncpy(speed, "8.0", PCI_SPEED_SIZE); break;
13816                 case i40e_bus_speed_5000:
13817                         strncpy(speed, "5.0", PCI_SPEED_SIZE); break;
13818                 case i40e_bus_speed_2500:
13819                         strncpy(speed, "2.5", PCI_SPEED_SIZE); break;
13820                 default:
13821                         break;
13822                 }
13823                 switch (hw->bus.width) {
13824                 case i40e_bus_width_pcie_x8:
13825                         strncpy(width, "8", PCI_WIDTH_SIZE); break;
13826                 case i40e_bus_width_pcie_x4:
13827                         strncpy(width, "4", PCI_WIDTH_SIZE); break;
13828                 case i40e_bus_width_pcie_x2:
13829                         strncpy(width, "2", PCI_WIDTH_SIZE); break;
13830                 case i40e_bus_width_pcie_x1:
13831                         strncpy(width, "1", PCI_WIDTH_SIZE); break;
13832                 default:
13833                         break;
13834                 }
13835
13836                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
13837                          speed, width);
13838
13839                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
13840                     hw->bus.speed < i40e_bus_speed_8000) {
13841                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
13842                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
13843                 }
13844         }
13845
13846         /* get the requested speeds from the fw */
13847         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
13848         if (err)
13849                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %s last_status =  %s\n",
13850                         i40e_stat_str(&pf->hw, err),
13851                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13852         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
13853
13854         /* get the supported phy types from the fw */
13855         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
13856         if (err)
13857                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %s last_status =  %s\n",
13858                         i40e_stat_str(&pf->hw, err),
13859                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13860
13861         /* Add a filter to drop all Flow control frames from any VSI from being
13862          * transmitted. By doing so we stop a malicious VF from sending out
13863          * PAUSE or PFC frames and potentially controlling traffic for other
13864          * PF/VF VSIs.
13865          * The FW can still send Flow control frames if enabled.
13866          */
13867         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
13868                                                        pf->main_vsi_seid);
13869
13870         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
13871                 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
13872                 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
13873         if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
13874                 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
13875         /* print a string summarizing features */
13876         i40e_print_features(pf);
13877
13878         return 0;
13879
13880         /* Unwind what we've done if something failed in the setup */
13881 err_vsis:
13882         set_bit(__I40E_DOWN, pf->state);
13883         i40e_clear_interrupt_scheme(pf);
13884         kfree(pf->vsi);
13885 err_switch_setup:
13886         i40e_reset_interrupt_capability(pf);
13887         del_timer_sync(&pf->service_timer);
13888 err_mac_addr:
13889 err_configure_lan_hmc:
13890         (void)i40e_shutdown_lan_hmc(hw);
13891 err_init_lan_hmc:
13892         kfree(pf->qp_pile);
13893 err_sw_init:
13894 err_adminq_setup:
13895 err_pf_reset:
13896         iounmap(hw->hw_addr);
13897 err_ioremap:
13898         kfree(pf);
13899 err_pf_alloc:
13900         pci_disable_pcie_error_reporting(pdev);
13901         pci_release_mem_regions(pdev);
13902 err_pci_reg:
13903 err_dma:
13904         pci_disable_device(pdev);
13905         return err;
13906 }
13907
13908 /**
13909  * i40e_remove - Device removal routine
13910  * @pdev: PCI device information struct
13911  *
13912  * i40e_remove is called by the PCI subsystem to alert the driver
13913  * that is should release a PCI device.  This could be caused by a
13914  * Hot-Plug event, or because the driver is going to be removed from
13915  * memory.
13916  **/
13917 static void i40e_remove(struct pci_dev *pdev)
13918 {
13919         struct i40e_pf *pf = pci_get_drvdata(pdev);
13920         struct i40e_hw *hw = &pf->hw;
13921         i40e_status ret_code;
13922         int i;
13923
13924         i40e_dbg_pf_exit(pf);
13925
13926         i40e_ptp_stop(pf);
13927
13928         /* Disable RSS in hw */
13929         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
13930         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
13931
13932         /* no more scheduling of any task */
13933         set_bit(__I40E_SUSPENDED, pf->state);
13934         set_bit(__I40E_DOWN, pf->state);
13935         if (pf->service_timer.function)
13936                 del_timer_sync(&pf->service_timer);
13937         if (pf->service_task.func)
13938                 cancel_work_sync(&pf->service_task);
13939
13940         /* Client close must be called explicitly here because the timer
13941          * has been stopped.
13942          */
13943         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
13944
13945         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
13946                 i40e_free_vfs(pf);
13947                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
13948         }
13949
13950         i40e_fdir_teardown(pf);
13951
13952         /* If there is a switch structure or any orphans, remove them.
13953          * This will leave only the PF's VSI remaining.
13954          */
13955         for (i = 0; i < I40E_MAX_VEB; i++) {
13956                 if (!pf->veb[i])
13957                         continue;
13958
13959                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
13960                     pf->veb[i]->uplink_seid == 0)
13961                         i40e_switch_branch_release(pf->veb[i]);
13962         }
13963
13964         /* Now we can shutdown the PF's VSI, just before we kill
13965          * adminq and hmc.
13966          */
13967         if (pf->vsi[pf->lan_vsi])
13968                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
13969
13970         i40e_cloud_filter_exit(pf);
13971
13972         /* remove attached clients */
13973         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
13974                 ret_code = i40e_lan_del_device(pf);
13975                 if (ret_code)
13976                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
13977                                  ret_code);
13978         }
13979
13980         /* shutdown and destroy the HMC */
13981         if (hw->hmc.hmc_obj) {
13982                 ret_code = i40e_shutdown_lan_hmc(hw);
13983                 if (ret_code)
13984                         dev_warn(&pdev->dev,
13985                                  "Failed to destroy the HMC resources: %d\n",
13986                                  ret_code);
13987         }
13988
13989         /* shutdown the adminq */
13990         i40e_shutdown_adminq(hw);
13991
13992         /* destroy the locks only once, here */
13993         mutex_destroy(&hw->aq.arq_mutex);
13994         mutex_destroy(&hw->aq.asq_mutex);
13995
13996         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
13997         i40e_clear_interrupt_scheme(pf);
13998         for (i = 0; i < pf->num_alloc_vsi; i++) {
13999                 if (pf->vsi[i]) {
14000                         i40e_vsi_clear_rings(pf->vsi[i]);
14001                         i40e_vsi_clear(pf->vsi[i]);
14002                         pf->vsi[i] = NULL;
14003                 }
14004         }
14005
14006         for (i = 0; i < I40E_MAX_VEB; i++) {
14007                 kfree(pf->veb[i]);
14008                 pf->veb[i] = NULL;
14009         }
14010
14011         kfree(pf->qp_pile);
14012         kfree(pf->vsi);
14013
14014         iounmap(hw->hw_addr);
14015         kfree(pf);
14016         pci_release_mem_regions(pdev);
14017
14018         pci_disable_pcie_error_reporting(pdev);
14019         pci_disable_device(pdev);
14020 }
14021
14022 /**
14023  * i40e_pci_error_detected - warning that something funky happened in PCI land
14024  * @pdev: PCI device information struct
14025  *
14026  * Called to warn that something happened and the error handling steps
14027  * are in progress.  Allows the driver to quiesce things, be ready for
14028  * remediation.
14029  **/
14030 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
14031                                                 enum pci_channel_state error)
14032 {
14033         struct i40e_pf *pf = pci_get_drvdata(pdev);
14034
14035         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
14036
14037         if (!pf) {
14038                 dev_info(&pdev->dev,
14039                          "Cannot recover - error happened during device probe\n");
14040                 return PCI_ERS_RESULT_DISCONNECT;
14041         }
14042
14043         /* shutdown all operations */
14044         if (!test_bit(__I40E_SUSPENDED, pf->state))
14045                 i40e_prep_for_reset(pf, false);
14046
14047         /* Request a slot reset */
14048         return PCI_ERS_RESULT_NEED_RESET;
14049 }
14050
14051 /**
14052  * i40e_pci_error_slot_reset - a PCI slot reset just happened
14053  * @pdev: PCI device information struct
14054  *
14055  * Called to find if the driver can work with the device now that
14056  * the pci slot has been reset.  If a basic connection seems good
14057  * (registers are readable and have sane content) then return a
14058  * happy little PCI_ERS_RESULT_xxx.
14059  **/
14060 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
14061 {
14062         struct i40e_pf *pf = pci_get_drvdata(pdev);
14063         pci_ers_result_t result;
14064         int err;
14065         u32 reg;
14066
14067         dev_dbg(&pdev->dev, "%s\n", __func__);
14068         if (pci_enable_device_mem(pdev)) {
14069                 dev_info(&pdev->dev,
14070                          "Cannot re-enable PCI device after reset.\n");
14071                 result = PCI_ERS_RESULT_DISCONNECT;
14072         } else {
14073                 pci_set_master(pdev);
14074                 pci_restore_state(pdev);
14075                 pci_save_state(pdev);
14076                 pci_wake_from_d3(pdev, false);
14077
14078                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
14079                 if (reg == 0)
14080                         result = PCI_ERS_RESULT_RECOVERED;
14081                 else
14082                         result = PCI_ERS_RESULT_DISCONNECT;
14083         }
14084
14085         err = pci_cleanup_aer_uncorrect_error_status(pdev);
14086         if (err) {
14087                 dev_info(&pdev->dev,
14088                          "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
14089                          err);
14090                 /* non-fatal, continue */
14091         }
14092
14093         return result;
14094 }
14095
14096 /**
14097  * i40e_pci_error_reset_prepare - prepare device driver for pci reset
14098  * @pdev: PCI device information struct
14099  */
14100 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
14101 {
14102         struct i40e_pf *pf = pci_get_drvdata(pdev);
14103
14104         i40e_prep_for_reset(pf, false);
14105 }
14106
14107 /**
14108  * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
14109  * @pdev: PCI device information struct
14110  */
14111 static void i40e_pci_error_reset_done(struct pci_dev *pdev)
14112 {
14113         struct i40e_pf *pf = pci_get_drvdata(pdev);
14114
14115         i40e_reset_and_rebuild(pf, false, false);
14116 }
14117
14118 /**
14119  * i40e_pci_error_resume - restart operations after PCI error recovery
14120  * @pdev: PCI device information struct
14121  *
14122  * Called to allow the driver to bring things back up after PCI error
14123  * and/or reset recovery has finished.
14124  **/
14125 static void i40e_pci_error_resume(struct pci_dev *pdev)
14126 {
14127         struct i40e_pf *pf = pci_get_drvdata(pdev);
14128
14129         dev_dbg(&pdev->dev, "%s\n", __func__);
14130         if (test_bit(__I40E_SUSPENDED, pf->state))
14131                 return;
14132
14133         i40e_handle_reset_warning(pf, false);
14134 }
14135
14136 /**
14137  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
14138  * using the mac_address_write admin q function
14139  * @pf: pointer to i40e_pf struct
14140  **/
14141 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
14142 {
14143         struct i40e_hw *hw = &pf->hw;
14144         i40e_status ret;
14145         u8 mac_addr[6];
14146         u16 flags = 0;
14147
14148         /* Get current MAC address in case it's an LAA */
14149         if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
14150                 ether_addr_copy(mac_addr,
14151                                 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
14152         } else {
14153                 dev_err(&pf->pdev->dev,
14154                         "Failed to retrieve MAC address; using default\n");
14155                 ether_addr_copy(mac_addr, hw->mac.addr);
14156         }
14157
14158         /* The FW expects the mac address write cmd to first be called with
14159          * one of these flags before calling it again with the multicast
14160          * enable flags.
14161          */
14162         flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
14163
14164         if (hw->func_caps.flex10_enable && hw->partition_id != 1)
14165                 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
14166
14167         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
14168         if (ret) {
14169                 dev_err(&pf->pdev->dev,
14170                         "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
14171                 return;
14172         }
14173
14174         flags = I40E_AQC_MC_MAG_EN
14175                         | I40E_AQC_WOL_PRESERVE_ON_PFR
14176                         | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
14177         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
14178         if (ret)
14179                 dev_err(&pf->pdev->dev,
14180                         "Failed to enable Multicast Magic Packet wake up\n");
14181 }
14182
14183 /**
14184  * i40e_shutdown - PCI callback for shutting down
14185  * @pdev: PCI device information struct
14186  **/
14187 static void i40e_shutdown(struct pci_dev *pdev)
14188 {
14189         struct i40e_pf *pf = pci_get_drvdata(pdev);
14190         struct i40e_hw *hw = &pf->hw;
14191
14192         set_bit(__I40E_SUSPENDED, pf->state);
14193         set_bit(__I40E_DOWN, pf->state);
14194         rtnl_lock();
14195         i40e_prep_for_reset(pf, true);
14196         rtnl_unlock();
14197
14198         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
14199         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
14200
14201         del_timer_sync(&pf->service_timer);
14202         cancel_work_sync(&pf->service_task);
14203         i40e_cloud_filter_exit(pf);
14204         i40e_fdir_teardown(pf);
14205
14206         /* Client close must be called explicitly here because the timer
14207          * has been stopped.
14208          */
14209         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
14210
14211         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
14212                 i40e_enable_mc_magic_wake(pf);
14213
14214         i40e_prep_for_reset(pf, false);
14215
14216         wr32(hw, I40E_PFPM_APM,
14217              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
14218         wr32(hw, I40E_PFPM_WUFC,
14219              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
14220
14221         i40e_clear_interrupt_scheme(pf);
14222
14223         if (system_state == SYSTEM_POWER_OFF) {
14224                 pci_wake_from_d3(pdev, pf->wol_en);
14225                 pci_set_power_state(pdev, PCI_D3hot);
14226         }
14227 }
14228
14229 /**
14230  * i40e_suspend - PM callback for moving to D3
14231  * @dev: generic device information structure
14232  **/
14233 static int __maybe_unused i40e_suspend(struct device *dev)
14234 {
14235         struct pci_dev *pdev = to_pci_dev(dev);
14236         struct i40e_pf *pf = pci_get_drvdata(pdev);
14237         struct i40e_hw *hw = &pf->hw;
14238
14239         /* If we're already suspended, then there is nothing to do */
14240         if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
14241                 return 0;
14242
14243         set_bit(__I40E_DOWN, pf->state);
14244
14245         /* Ensure service task will not be running */
14246         del_timer_sync(&pf->service_timer);
14247         cancel_work_sync(&pf->service_task);
14248
14249         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
14250                 i40e_enable_mc_magic_wake(pf);
14251
14252         i40e_prep_for_reset(pf, false);
14253
14254         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
14255         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
14256
14257         /* Clear the interrupt scheme and release our IRQs so that the system
14258          * can safely hibernate even when there are a large number of CPUs.
14259          * Otherwise hibernation might fail when mapping all the vectors back
14260          * to CPU0.
14261          */
14262         i40e_clear_interrupt_scheme(pf);
14263
14264         return 0;
14265 }
14266
14267 /**
14268  * i40e_resume - PM callback for waking up from D3
14269  * @dev: generic device information structure
14270  **/
14271 static int __maybe_unused i40e_resume(struct device *dev)
14272 {
14273         struct pci_dev *pdev = to_pci_dev(dev);
14274         struct i40e_pf *pf = pci_get_drvdata(pdev);
14275         int err;
14276
14277         /* If we're not suspended, then there is nothing to do */
14278         if (!test_bit(__I40E_SUSPENDED, pf->state))
14279                 return 0;
14280
14281         /* We cleared the interrupt scheme when we suspended, so we need to
14282          * restore it now to resume device functionality.
14283          */
14284         err = i40e_restore_interrupt_scheme(pf);
14285         if (err) {
14286                 dev_err(&pdev->dev, "Cannot restore interrupt scheme: %d\n",
14287                         err);
14288         }
14289
14290         clear_bit(__I40E_DOWN, pf->state);
14291         i40e_reset_and_rebuild(pf, false, false);
14292
14293         /* Clear suspended state last after everything is recovered */
14294         clear_bit(__I40E_SUSPENDED, pf->state);
14295
14296         /* Restart the service task */
14297         mod_timer(&pf->service_timer,
14298                   round_jiffies(jiffies + pf->service_timer_period));
14299
14300         return 0;
14301 }
14302
14303 static const struct pci_error_handlers i40e_err_handler = {
14304         .error_detected = i40e_pci_error_detected,
14305         .slot_reset = i40e_pci_error_slot_reset,
14306         .reset_prepare = i40e_pci_error_reset_prepare,
14307         .reset_done = i40e_pci_error_reset_done,
14308         .resume = i40e_pci_error_resume,
14309 };
14310
14311 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
14312
14313 static struct pci_driver i40e_driver = {
14314         .name     = i40e_driver_name,
14315         .id_table = i40e_pci_tbl,
14316         .probe    = i40e_probe,
14317         .remove   = i40e_remove,
14318         .driver   = {
14319                 .pm = &i40e_pm_ops,
14320         },
14321         .shutdown = i40e_shutdown,
14322         .err_handler = &i40e_err_handler,
14323         .sriov_configure = i40e_pci_sriov_configure,
14324 };
14325
14326 /**
14327  * i40e_init_module - Driver registration routine
14328  *
14329  * i40e_init_module is the first routine called when the driver is
14330  * loaded. All it does is register with the PCI subsystem.
14331  **/
14332 static int __init i40e_init_module(void)
14333 {
14334         pr_info("%s: %s - version %s\n", i40e_driver_name,
14335                 i40e_driver_string, i40e_driver_version_str);
14336         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
14337
14338         /* There is no need to throttle the number of active tasks because
14339          * each device limits its own task using a state bit for scheduling
14340          * the service task, and the device tasks do not interfere with each
14341          * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
14342          * since we need to be able to guarantee forward progress even under
14343          * memory pressure.
14344          */
14345         i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
14346         if (!i40e_wq) {
14347                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
14348                 return -ENOMEM;
14349         }
14350
14351         i40e_dbg_init();
14352         return pci_register_driver(&i40e_driver);
14353 }
14354 module_init(i40e_init_module);
14355
14356 /**
14357  * i40e_exit_module - Driver exit cleanup routine
14358  *
14359  * i40e_exit_module is called just before the driver is removed
14360  * from memory.
14361  **/
14362 static void __exit i40e_exit_module(void)
14363 {
14364         pci_unregister_driver(&i40e_driver);
14365         destroy_workqueue(i40e_wq);
14366         i40e_dbg_exit();
14367 }
14368 module_exit(i40e_exit_module);