drivers/net/ethernet/qlogic/qede/qede_main.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
   2 /* QLogic qede NIC Driver
   3  * Copyright (c) 2015-2017  QLogic Corporation
   4  * Copyright (c) 2019-2020 Marvell International Ltd.
   5  */
   6
   7 #include <linux/crash_dump.h>
   8 #include <linux/module.h>
   9 #include <linux/pci.h>
  10 #include <linux/device.h>
  11 #include <linux/netdevice.h>
  12 #include <linux/etherdevice.h>
  13 #include <linux/skbuff.h>
  14 #include <linux/errno.h>
  15 #include <linux/list.h>
  16 #include <linux/string.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/interrupt.h>
  19 #include <asm/byteorder.h>
  20 #include <asm/param.h>
  21 #include <linux/io.h>
  22 #include <linux/netdev_features.h>
  23 #include <linux/udp.h>
  24 #include <linux/tcp.h>
  25 #include <net/udp_tunnel.h>
  26 #include <linux/ip.h>
  27 #include <net/ipv6.h>
  28 #include <net/tcp.h>
  29 #include <linux/if_ether.h>
  30 #include <linux/if_vlan.h>
  31 #include <linux/pkt_sched.h>
  32 #include <linux/ethtool.h>
  33 #include <linux/in.h>
  34 #include <linux/random.h>
  35 #include <net/ip6_checksum.h>
  36 #include <linux/bitops.h>
  37 #include <linux/vmalloc.h>
  38 #include "qede.h"
  39 #include "qede_ptp.h"
  40
  41 MODULE_DESCRIPTION("QLogic FastLinQ 4xxxx Ethernet Driver");
  42 MODULE_LICENSE("GPL");
  43
  44 static uint debug;
  45 module_param(debug, uint, 0);
  46 MODULE_PARM_DESC(debug, " Default debug msglevel");
  47
  48 static const struct qed_eth_ops *qed_ops;
  49
  50 #define CHIP_NUM_57980S_40              0x1634
  51 #define CHIP_NUM_57980S_10              0x1666
  52 #define CHIP_NUM_57980S_MF              0x1636
  53 #define CHIP_NUM_57980S_100             0x1644
  54 #define CHIP_NUM_57980S_50              0x1654
  55 #define CHIP_NUM_57980S_25              0x1656
  56 #define CHIP_NUM_57980S_IOV             0x1664
  57 #define CHIP_NUM_AH                     0x8070
  58 #define CHIP_NUM_AH_IOV                 0x8090
  59
  60 #ifndef PCI_DEVICE_ID_NX2_57980E
  61 #define PCI_DEVICE_ID_57980S_40         CHIP_NUM_57980S_40
  62 #define PCI_DEVICE_ID_57980S_10         CHIP_NUM_57980S_10
  63 #define PCI_DEVICE_ID_57980S_MF         CHIP_NUM_57980S_MF
  64 #define PCI_DEVICE_ID_57980S_100        CHIP_NUM_57980S_100
  65 #define PCI_DEVICE_ID_57980S_50         CHIP_NUM_57980S_50
  66 #define PCI_DEVICE_ID_57980S_25         CHIP_NUM_57980S_25
  67 #define PCI_DEVICE_ID_57980S_IOV        CHIP_NUM_57980S_IOV
  68 #define PCI_DEVICE_ID_AH                CHIP_NUM_AH
  69 #define PCI_DEVICE_ID_AH_IOV            CHIP_NUM_AH_IOV
  70
  71 #endif
  72
  73 enum qede_pci_private {
  74         QEDE_PRIVATE_PF,
  75         QEDE_PRIVATE_VF
  76 };
  77
  78 static const struct pci_device_id qede_pci_tbl[] = {
  79         {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_40), QEDE_PRIVATE_PF},
  80         {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_10), QEDE_PRIVATE_PF},
  81         {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_MF), QEDE_PRIVATE_PF},
  82         {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_100), QEDE_PRIVATE_PF},
  83         {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_50), QEDE_PRIVATE_PF},
  84         {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_25), QEDE_PRIVATE_PF},
  85 #ifdef CONFIG_QED_SRIOV
  86         {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_IOV), QEDE_PRIVATE_VF},
  87 #endif
  88         {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_AH), QEDE_PRIVATE_PF},
  89 #ifdef CONFIG_QED_SRIOV
  90         {PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_AH_IOV), QEDE_PRIVATE_VF},
  91 #endif
  92         { 0 }
  93 };
  94
  95 MODULE_DEVICE_TABLE(pci, qede_pci_tbl);
  96
  97 static int qede_probe(struct pci_dev *pdev, const struct pci_device_id *id);
  98 static pci_ers_result_t
  99 qede_io_error_detected(struct pci_dev *pdev, pci_channel_state_t state);
 100
 101 #define TX_TIMEOUT              (5 * HZ)
 102
 103 /* Utilize last protocol index for XDP */
 104 #define XDP_PI  11
 105
 106 static void qede_remove(struct pci_dev *pdev);
 107 static void qede_shutdown(struct pci_dev *pdev);
 108 static void qede_link_update(void *dev, struct qed_link_output *link);
 109 static void qede_schedule_recovery_handler(void *dev);
 110 static void qede_recovery_handler(struct qede_dev *edev);
 111 static void qede_schedule_hw_err_handler(void *dev,
 112                                          enum qed_hw_err_type err_type);
 113 static void qede_get_eth_tlv_data(void *edev, void *data);
 114 static void qede_get_generic_tlv_data(void *edev,
 115                                       struct qed_generic_tlvs *data);
 116 static void qede_generic_hw_err_handler(struct qede_dev *edev);
 117 #ifdef CONFIG_QED_SRIOV
 118 static int qede_set_vf_vlan(struct net_device *ndev, int vf, u16 vlan, u8 qos,
 119                             __be16 vlan_proto)
 120 {
 121         struct qede_dev *edev = netdev_priv(ndev);
 122
 123         if (vlan > 4095) {
 124                 DP_NOTICE(edev, "Illegal vlan value %d\n", vlan);
 125                 return -EINVAL;
 126         }
 127
 128         if (vlan_proto != htons(ETH_P_8021Q))
 129                 return -EPROTONOSUPPORT;
 130
 131         DP_VERBOSE(edev, QED_MSG_IOV, "Setting Vlan 0x%04x to VF [%d]\n",
 132                    vlan, vf);
 133
 134         return edev->ops->iov->set_vlan(edev->cdev, vlan, vf);
 135 }
 136
 137 static int qede_set_vf_mac(struct net_device *ndev, int vfidx, u8 *mac)
 138 {
 139         struct qede_dev *edev = netdev_priv(ndev);
 140
 141         DP_VERBOSE(edev, QED_MSG_IOV, "Setting MAC %pM to VF [%d]\n", mac, vfidx);
 142
 143         if (!is_valid_ether_addr(mac)) {
 144                 DP_VERBOSE(edev, QED_MSG_IOV, "MAC address isn't valid\n");
 145                 return -EINVAL;
 146         }
 147
 148         return edev->ops->iov->set_mac(edev->cdev, mac, vfidx);
 149 }
 150
 151 static int qede_sriov_configure(struct pci_dev *pdev, int num_vfs_param)
 152 {
 153         struct qede_dev *edev = netdev_priv(pci_get_drvdata(pdev));
 154         struct qed_dev_info *qed_info = &edev->dev_info.common;
 155         struct qed_update_vport_params *vport_params;
 156         int rc;
 157
 158         vport_params = vzalloc(sizeof(*vport_params));
 159         if (!vport_params)
 160                 return -ENOMEM;
 161         DP_VERBOSE(edev, QED_MSG_IOV, "Requested %d VFs\n", num_vfs_param);
 162
 163         rc = edev->ops->iov->configure(edev->cdev, num_vfs_param);
 164
 165         /* Enable/Disable Tx switching for PF */
 166         if ((rc == num_vfs_param) && netif_running(edev->ndev) &&
 167             !qed_info->b_inter_pf_switch && qed_info->tx_switching) {
 168                 vport_params->vport_id = 0;
 169                 vport_params->update_tx_switching_flg = 1;
 170                 vport_params->tx_switching_flg = num_vfs_param ? 1 : 0;
 171                 edev->ops->vport_update(edev->cdev, vport_params);
 172         }
 173
 174         vfree(vport_params);
 175         return rc;
 176 }
 177 #endif
 178
 179 static int __maybe_unused qede_suspend(struct device *dev)
 180 {
 181         dev_info(dev, "Device does not support suspend operation\n");
 182
 183         return -EOPNOTSUPP;
 184 }
 185
 186 static DEFINE_SIMPLE_DEV_PM_OPS(qede_pm_ops, qede_suspend, NULL);
 187
 188 static const struct pci_error_handlers qede_err_handler = {
 189         .error_detected = qede_io_error_detected,
 190 };
 191
 192 static struct pci_driver qede_pci_driver = {
 193         .name = "qede",
 194         .id_table = qede_pci_tbl,
 195         .probe = qede_probe,
 196         .remove = qede_remove,
 197         .shutdown = qede_shutdown,
 198 #ifdef CONFIG_QED_SRIOV
 199         .sriov_configure = qede_sriov_configure,
 200 #endif
 201         .err_handler = &qede_err_handler,
 202         .driver.pm = &qede_pm_ops,
 203 };
 204
 205 static struct qed_eth_cb_ops qede_ll_ops = {
 206         {
 207 #ifdef CONFIG_RFS_ACCEL
 208                 .arfs_filter_op = qede_arfs_filter_op,
 209 #endif
 210                 .link_update = qede_link_update,
 211                 .schedule_recovery_handler = qede_schedule_recovery_handler,
 212                 .schedule_hw_err_handler = qede_schedule_hw_err_handler,
 213                 .get_generic_tlv_data = qede_get_generic_tlv_data,
 214                 .get_protocol_tlv_data = qede_get_eth_tlv_data,
 215         },
 216         .force_mac = qede_force_mac,
 217         .ports_update = qede_udp_ports_update,
 218 };
 219
 220 static int qede_netdev_event(struct notifier_block *this, unsigned long event,
 221                              void *ptr)
 222 {
 223         struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
 224         struct ethtool_drvinfo drvinfo;
 225         struct qede_dev *edev;
 226
 227         if (event != NETDEV_CHANGENAME && event != NETDEV_CHANGEADDR)
 228                 goto done;
 229
 230         /* Check whether this is a qede device */
 231         if (!ndev || !ndev->ethtool_ops || !ndev->ethtool_ops->get_drvinfo)
 232                 goto done;
 233
 234         memset(&drvinfo, 0, sizeof(drvinfo));
 235         ndev->ethtool_ops->get_drvinfo(ndev, &drvinfo);
 236         if (strcmp(drvinfo.driver, "qede"))
 237                 goto done;
 238         edev = netdev_priv(ndev);
 239
 240         switch (event) {
 241         case NETDEV_CHANGENAME:
 242                 /* Notify qed of the name change */
 243                 if (!edev->ops || !edev->ops->common)
 244                         goto done;
 245                 edev->ops->common->set_name(edev->cdev, edev->ndev->name);
 246                 break;
 247         case NETDEV_CHANGEADDR:
 248                 edev = netdev_priv(ndev);
 249                 qede_rdma_event_changeaddr(edev);
 250                 break;
 251         }
 252
 253 done:
 254         return NOTIFY_DONE;
 255 }
 256
 257 static struct notifier_block qede_netdev_notifier = {
 258         .notifier_call = qede_netdev_event,
 259 };
 260
 261 static
 262 int __init qede_init(void)
 263 {
 264         int ret;
 265
 266         pr_info("qede init: QLogic FastLinQ 4xxxx Ethernet Driver qede\n");
 267
 268         qede_forced_speed_maps_init();
 269
 270         qed_ops = qed_get_eth_ops();
 271         if (!qed_ops) {
 272                 pr_notice("Failed to get qed ethtool operations\n");
 273                 return -EINVAL;
 274         }
 275
 276         /* Must register notifier before pci ops, since we might miss
 277          * interface rename after pci probe and netdev registration.
 278          */
 279         ret = register_netdevice_notifier(&qede_netdev_notifier);
 280         if (ret) {
 281                 pr_notice("Failed to register netdevice_notifier\n");
 282                 qed_put_eth_ops();
 283                 return -EINVAL;
 284         }
 285
 286         ret = pci_register_driver(&qede_pci_driver);
 287         if (ret) {
 288                 pr_notice("Failed to register driver\n");
 289                 unregister_netdevice_notifier(&qede_netdev_notifier);
 290                 qed_put_eth_ops();
 291                 return -EINVAL;
 292         }
 293
 294         return 0;
 295 }
 296
 297 static void __exit qede_cleanup(void)
 298 {
 299         if (debug & QED_LOG_INFO_MASK)
 300                 pr_info("qede_cleanup called\n");
 301
 302         unregister_netdevice_notifier(&qede_netdev_notifier);
 303         pci_unregister_driver(&qede_pci_driver);
 304         qed_put_eth_ops();
 305 }
 306
 307 module_init(qede_init);
 308 module_exit(qede_cleanup);
 309
 310 static int qede_open(struct net_device *ndev);
 311 static int qede_close(struct net_device *ndev);
 312
 313 void qede_fill_by_demand_stats(struct qede_dev *edev)
 314 {
 315         struct qede_stats_common *p_common = &edev->stats.common;
 316         struct qed_eth_stats stats;
 317
 318         edev->ops->get_vport_stats(edev->cdev, &stats);
 319
 320         spin_lock(&edev->stats_lock);
 321
 322         p_common->no_buff_discards = stats.common.no_buff_discards;
 323         p_common->packet_too_big_discard = stats.common.packet_too_big_discard;
 324         p_common->ttl0_discard = stats.common.ttl0_discard;
 325         p_common->rx_ucast_bytes = stats.common.rx_ucast_bytes;
 326         p_common->rx_mcast_bytes = stats.common.rx_mcast_bytes;
 327         p_common->rx_bcast_bytes = stats.common.rx_bcast_bytes;
 328         p_common->rx_ucast_pkts = stats.common.rx_ucast_pkts;
 329         p_common->rx_mcast_pkts = stats.common.rx_mcast_pkts;
 330         p_common->rx_bcast_pkts = stats.common.rx_bcast_pkts;
 331         p_common->mftag_filter_discards = stats.common.mftag_filter_discards;
 332         p_common->mac_filter_discards = stats.common.mac_filter_discards;
 333         p_common->gft_filter_drop = stats.common.gft_filter_drop;
 334
 335         p_common->tx_ucast_bytes = stats.common.tx_ucast_bytes;
 336         p_common->tx_mcast_bytes = stats.common.tx_mcast_bytes;
 337         p_common->tx_bcast_bytes = stats.common.tx_bcast_bytes;
 338         p_common->tx_ucast_pkts = stats.common.tx_ucast_pkts;
 339         p_common->tx_mcast_pkts = stats.common.tx_mcast_pkts;
 340         p_common->tx_bcast_pkts = stats.common.tx_bcast_pkts;
 341         p_common->tx_err_drop_pkts = stats.common.tx_err_drop_pkts;
 342         p_common->coalesced_pkts = stats.common.tpa_coalesced_pkts;
 343         p_common->coalesced_events = stats.common.tpa_coalesced_events;
 344         p_common->coalesced_aborts_num = stats.common.tpa_aborts_num;
 345         p_common->non_coalesced_pkts = stats.common.tpa_not_coalesced_pkts;
 346         p_common->coalesced_bytes = stats.common.tpa_coalesced_bytes;
 347
 348         p_common->rx_64_byte_packets = stats.common.rx_64_byte_packets;
 349         p_common->rx_65_to_127_byte_packets =
 350             stats.common.rx_65_to_127_byte_packets;
 351         p_common->rx_128_to_255_byte_packets =
 352             stats.common.rx_128_to_255_byte_packets;
 353         p_common->rx_256_to_511_byte_packets =
 354             stats.common.rx_256_to_511_byte_packets;
 355         p_common->rx_512_to_1023_byte_packets =
 356             stats.common.rx_512_to_1023_byte_packets;
 357         p_common->rx_1024_to_1518_byte_packets =
 358             stats.common.rx_1024_to_1518_byte_packets;
 359         p_common->rx_crc_errors = stats.common.rx_crc_errors;
 360         p_common->rx_mac_crtl_frames = stats.common.rx_mac_crtl_frames;
 361         p_common->rx_pause_frames = stats.common.rx_pause_frames;
 362         p_common->rx_pfc_frames = stats.common.rx_pfc_frames;
 363         p_common->rx_align_errors = stats.common.rx_align_errors;
 364         p_common->rx_carrier_errors = stats.common.rx_carrier_errors;
 365         p_common->rx_oversize_packets = stats.common.rx_oversize_packets;
 366         p_common->rx_jabbers = stats.common.rx_jabbers;
 367         p_common->rx_undersize_packets = stats.common.rx_undersize_packets;
 368         p_common->rx_fragments = stats.common.rx_fragments;
 369         p_common->tx_64_byte_packets = stats.common.tx_64_byte_packets;
 370         p_common->tx_65_to_127_byte_packets =
 371             stats.common.tx_65_to_127_byte_packets;
 372         p_common->tx_128_to_255_byte_packets =
 373             stats.common.tx_128_to_255_byte_packets;
 374         p_common->tx_256_to_511_byte_packets =
 375             stats.common.tx_256_to_511_byte_packets;
 376         p_common->tx_512_to_1023_byte_packets =
 377             stats.common.tx_512_to_1023_byte_packets;
 378         p_common->tx_1024_to_1518_byte_packets =
 379             stats.common.tx_1024_to_1518_byte_packets;
 380         p_common->tx_pause_frames = stats.common.tx_pause_frames;
 381         p_common->tx_pfc_frames = stats.common.tx_pfc_frames;
 382         p_common->brb_truncates = stats.common.brb_truncates;
 383         p_common->brb_discards = stats.common.brb_discards;
 384         p_common->tx_mac_ctrl_frames = stats.common.tx_mac_ctrl_frames;
 385         p_common->link_change_count = stats.common.link_change_count;
 386         p_common->ptp_skip_txts = edev->ptp_skip_txts;
 387
 388         if (QEDE_IS_BB(edev)) {
 389                 struct qede_stats_bb *p_bb = &edev->stats.bb;
 390
 391                 p_bb->rx_1519_to_1522_byte_packets =
 392                     stats.bb.rx_1519_to_1522_byte_packets;
 393                 p_bb->rx_1519_to_2047_byte_packets =
 394                     stats.bb.rx_1519_to_2047_byte_packets;
 395                 p_bb->rx_2048_to_4095_byte_packets =
 396                     stats.bb.rx_2048_to_4095_byte_packets;
 397                 p_bb->rx_4096_to_9216_byte_packets =
 398                     stats.bb.rx_4096_to_9216_byte_packets;
 399                 p_bb->rx_9217_to_16383_byte_packets =
 400                     stats.bb.rx_9217_to_16383_byte_packets;
 401                 p_bb->tx_1519_to_2047_byte_packets =
 402                     stats.bb.tx_1519_to_2047_byte_packets;
 403                 p_bb->tx_2048_to_4095_byte_packets =
 404                     stats.bb.tx_2048_to_4095_byte_packets;
 405                 p_bb->tx_4096_to_9216_byte_packets =
 406                     stats.bb.tx_4096_to_9216_byte_packets;
 407                 p_bb->tx_9217_to_16383_byte_packets =
 408                     stats.bb.tx_9217_to_16383_byte_packets;
 409                 p_bb->tx_lpi_entry_count = stats.bb.tx_lpi_entry_count;
 410                 p_bb->tx_total_collisions = stats.bb.tx_total_collisions;
 411         } else {
 412                 struct qede_stats_ah *p_ah = &edev->stats.ah;
 413
 414                 p_ah->rx_1519_to_max_byte_packets =
 415                     stats.ah.rx_1519_to_max_byte_packets;
 416                 p_ah->tx_1519_to_max_byte_packets =
 417                     stats.ah.tx_1519_to_max_byte_packets;
 418         }
 419
 420         spin_unlock(&edev->stats_lock);
 421 }
 422
 423 static void qede_get_stats64(struct net_device *dev,
 424                              struct rtnl_link_stats64 *stats)
 425 {
 426         struct qede_dev *edev = netdev_priv(dev);
 427         struct qede_stats_common *p_common;
 428
 429         p_common = &edev->stats.common;
 430
 431         spin_lock(&edev->stats_lock);
 432
 433         stats->rx_packets = p_common->rx_ucast_pkts + p_common->rx_mcast_pkts +
 434                             p_common->rx_bcast_pkts;
 435         stats->tx_packets = p_common->tx_ucast_pkts + p_common->tx_mcast_pkts +
 436                             p_common->tx_bcast_pkts;
 437
 438         stats->rx_bytes = p_common->rx_ucast_bytes + p_common->rx_mcast_bytes +
 439                           p_common->rx_bcast_bytes;
 440         stats->tx_bytes = p_common->tx_ucast_bytes + p_common->tx_mcast_bytes +
 441                           p_common->tx_bcast_bytes;
 442
 443         stats->tx_errors = p_common->tx_err_drop_pkts;
 444         stats->multicast = p_common->rx_mcast_pkts + p_common->rx_bcast_pkts;
 445
 446         stats->rx_fifo_errors = p_common->no_buff_discards;
 447
 448         if (QEDE_IS_BB(edev))
 449                 stats->collisions = edev->stats.bb.tx_total_collisions;
 450         stats->rx_crc_errors = p_common->rx_crc_errors;
 451         stats->rx_frame_errors = p_common->rx_align_errors;
 452
 453         spin_unlock(&edev->stats_lock);
 454 }
 455
 456 #ifdef CONFIG_QED_SRIOV
 457 static int qede_get_vf_config(struct net_device *dev, int vfidx,
 458                               struct ifla_vf_info *ivi)
 459 {
 460         struct qede_dev *edev = netdev_priv(dev);
 461
 462         if (!edev->ops)
 463                 return -EINVAL;
 464
 465         return edev->ops->iov->get_config(edev->cdev, vfidx, ivi);
 466 }
 467
 468 static int qede_set_vf_rate(struct net_device *dev, int vfidx,
 469                             int min_tx_rate, int max_tx_rate)
 470 {
 471         struct qede_dev *edev = netdev_priv(dev);
 472
 473         return edev->ops->iov->set_rate(edev->cdev, vfidx, min_tx_rate,
 474                                         max_tx_rate);
 475 }
 476
 477 static int qede_set_vf_spoofchk(struct net_device *dev, int vfidx, bool val)
 478 {
 479         struct qede_dev *edev = netdev_priv(dev);
 480
 481         if (!edev->ops)
 482                 return -EINVAL;
 483
 484         return edev->ops->iov->set_spoof(edev->cdev, vfidx, val);
 485 }
 486
 487 static int qede_set_vf_link_state(struct net_device *dev, int vfidx,
 488                                   int link_state)
 489 {
 490         struct qede_dev *edev = netdev_priv(dev);
 491
 492         if (!edev->ops)
 493                 return -EINVAL;
 494
 495         return edev->ops->iov->set_link_state(edev->cdev, vfidx, link_state);
 496 }
 497
 498 static int qede_set_vf_trust(struct net_device *dev, int vfidx, bool setting)
 499 {
 500         struct qede_dev *edev = netdev_priv(dev);
 501
 502         if (!edev->ops)
 503                 return -EINVAL;
 504
 505         return edev->ops->iov->set_trust(edev->cdev, vfidx, setting);
 506 }
 507 #endif
 508
 509 static int qede_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 510 {
 511         struct qede_dev *edev = netdev_priv(dev);
 512
 513         if (!netif_running(dev))
 514                 return -EAGAIN;
 515
 516         switch (cmd) {
 517         case SIOCSHWTSTAMP:
 518                 return qede_ptp_hw_ts(edev, ifr);
 519         default:
 520                 DP_VERBOSE(edev, QED_MSG_DEBUG,
 521                            "default IOCTL cmd 0x%x\n", cmd);
 522                 return -EOPNOTSUPP;
 523         }
 524
 525         return 0;
 526 }
 527
 528 static void qede_fp_sb_dump(struct qede_dev *edev, struct qede_fastpath *fp)
 529 {
 530         char *p_sb = (char *)fp->sb_info->sb_virt;
 531         u32 sb_size, i;
 532
 533         sb_size = sizeof(struct status_block);
 534
 535         for (i = 0; i < sb_size; i += 8)
 536                 DP_NOTICE(edev,
 537                           "%02hhX %02hhX %02hhX %02hhX  %02hhX %02hhX %02hhX %02hhX\n",
 538                           p_sb[i], p_sb[i + 1], p_sb[i + 2], p_sb[i + 3],
 539                           p_sb[i + 4], p_sb[i + 5], p_sb[i + 6], p_sb[i + 7]);
 540 }
 541
 542 static void
 543 qede_txq_fp_log_metadata(struct qede_dev *edev,
 544                          struct qede_fastpath *fp, struct qede_tx_queue *txq)
 545 {
 546         struct qed_chain *p_chain = &txq->tx_pbl;
 547
 548         /* Dump txq/fp/sb ids etc. other metadata */
 549         DP_NOTICE(edev,
 550                   "fpid 0x%x sbid 0x%x txqid [0x%x] ndev_qid [0x%x] cos [0x%x] p_chain %p cap %d size %d jiffies %lu HZ 0x%x\n",
 551                   fp->id, fp->sb_info->igu_sb_id, txq->index, txq->ndev_txq_id, txq->cos,
 552                   p_chain, p_chain->capacity, p_chain->size, jiffies, HZ);
 553
 554         /* Dump all the relevant prod/cons indexes */
 555         DP_NOTICE(edev,
 556                   "hw cons %04x sw_tx_prod=0x%x, sw_tx_cons=0x%x, bd_prod 0x%x bd_cons 0x%x\n",
 557                   le16_to_cpu(*txq->hw_cons_ptr), txq->sw_tx_prod, txq->sw_tx_cons,
 558                   qed_chain_get_prod_idx(p_chain), qed_chain_get_cons_idx(p_chain));
 559 }
 560
 561 static void
 562 qede_tx_log_print(struct qede_dev *edev, struct qede_fastpath *fp, struct qede_tx_queue *txq)
 563 {
 564         struct qed_sb_info_dbg sb_dbg;
 565         int rc;
 566
 567         /* sb info */
 568         qede_fp_sb_dump(edev, fp);
 569
 570         memset(&sb_dbg, 0, sizeof(sb_dbg));
 571         rc = edev->ops->common->get_sb_info(edev->cdev, fp->sb_info, (u16)fp->id, &sb_dbg);
 572
 573         DP_NOTICE(edev, "IGU: prod %08x cons %08x CAU Tx %04x\n",
 574                   sb_dbg.igu_prod, sb_dbg.igu_cons, sb_dbg.pi[TX_PI(txq->cos)]);
 575
 576         /* report to mfw */
 577         edev->ops->common->mfw_report(edev->cdev,
 578                                       "Txq[%d]: FW cons [host] %04x, SW cons %04x, SW prod %04x [Jiffies %lu]\n",
 579                                       txq->index, le16_to_cpu(*txq->hw_cons_ptr),
 580                                       qed_chain_get_cons_idx(&txq->tx_pbl),
 581                                       qed_chain_get_prod_idx(&txq->tx_pbl), jiffies);
 582         if (!rc)
 583                 edev->ops->common->mfw_report(edev->cdev,
 584                                               "Txq[%d]: SB[0x%04x] - IGU: prod %08x cons %08x CAU Tx %04x\n",
 585                                               txq->index, fp->sb_info->igu_sb_id,
 586                                               sb_dbg.igu_prod, sb_dbg.igu_cons,
 587                                               sb_dbg.pi[TX_PI(txq->cos)]);
 588 }
 589
 590 static void qede_tx_timeout(struct net_device *dev, unsigned int txqueue)
 591 {
 592         struct qede_dev *edev = netdev_priv(dev);
 593         int i;
 594
 595         netif_carrier_off(dev);
 596         DP_NOTICE(edev, "TX timeout on queue %u!\n", txqueue);
 597
 598         for_each_queue(i) {
 599                 struct qede_tx_queue *txq;
 600                 struct qede_fastpath *fp;
 601                 int cos;
 602
 603                 fp = &edev->fp_array[i];
 604                 if (!(fp->type & QEDE_FASTPATH_TX))
 605                         continue;
 606
 607                 for_each_cos_in_txq(edev, cos) {
 608                         txq = &fp->txq[cos];
 609
 610                         /* Dump basic metadata for all queues */
 611                         qede_txq_fp_log_metadata(edev, fp, txq);
 612
 613                         if (qed_chain_get_cons_idx(&txq->tx_pbl) !=
 614                             qed_chain_get_prod_idx(&txq->tx_pbl))
 615                                 qede_tx_log_print(edev, fp, txq);
 616                 }
 617         }
 618
 619         if (IS_VF(edev))
 620                 return;
 621
 622         if (test_and_set_bit(QEDE_ERR_IS_HANDLED, &edev->err_flags) ||
 623             edev->state == QEDE_STATE_RECOVERY) {
 624                 DP_INFO(edev,
 625                         "Avoid handling a Tx timeout while another HW error is being handled\n");
 626                 return;
 627         }
 628
 629         set_bit(QEDE_ERR_GET_DBG_INFO, &edev->err_flags);
 630         set_bit(QEDE_SP_HW_ERR, &edev->sp_flags);
 631         schedule_delayed_work(&edev->sp_task, 0);
 632 }
 633
 634 static int qede_setup_tc(struct net_device *ndev, u8 num_tc)
 635 {
 636         struct qede_dev *edev = netdev_priv(ndev);
 637         int cos, count, offset;
 638
 639         if (num_tc > edev->dev_info.num_tc)
 640                 return -EINVAL;
 641
 642         netdev_reset_tc(ndev);
 643         netdev_set_num_tc(ndev, num_tc);
 644
 645         for_each_cos_in_txq(edev, cos) {
 646                 count = QEDE_TSS_COUNT(edev);
 647                 offset = cos * QEDE_TSS_COUNT(edev);
 648                 netdev_set_tc_queue(ndev, cos, count, offset);
 649         }
 650
 651         return 0;
 652 }
 653
 654 static int
 655 qede_set_flower(struct qede_dev *edev, struct flow_cls_offload *f,
 656                 __be16 proto)
 657 {
 658         switch (f->command) {
 659         case FLOW_CLS_REPLACE:
 660                 return qede_add_tc_flower_fltr(edev, proto, f);
 661         case FLOW_CLS_DESTROY:
 662                 return qede_delete_flow_filter(edev, f->cookie);
 663         default:
 664                 return -EOPNOTSUPP;
 665         }
 666 }
 667
 668 static int qede_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
 669                                   void *cb_priv)
 670 {
 671         struct flow_cls_offload *f;
 672         struct qede_dev *edev = cb_priv;
 673
 674         if (!tc_cls_can_offload_and_chain0(edev->ndev, type_data))
 675                 return -EOPNOTSUPP;
 676
 677         switch (type) {
 678         case TC_SETUP_CLSFLOWER:
 679                 f = type_data;
 680                 return qede_set_flower(edev, f, f->common.protocol);
 681         default:
 682                 return -EOPNOTSUPP;
 683         }
 684 }
 685
 686 static LIST_HEAD(qede_block_cb_list);
 687
 688 static int
 689 qede_setup_tc_offload(struct net_device *dev, enum tc_setup_type type,
 690                       void *type_data)
 691 {
 692         struct qede_dev *edev = netdev_priv(dev);
 693         struct tc_mqprio_qopt *mqprio;
 694
 695         switch (type) {
 696         case TC_SETUP_BLOCK:
 697                 return flow_block_cb_setup_simple(type_data,
 698                                                   &qede_block_cb_list,
 699                                                   qede_setup_tc_block_cb,
 700                                                   edev, edev, true);
 701         case TC_SETUP_QDISC_MQPRIO:
 702                 mqprio = type_data;
 703
 704                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
 705                 return qede_setup_tc(dev, mqprio->num_tc);
 706         default:
 707                 return -EOPNOTSUPP;
 708         }
 709 }
 710
 711 static const struct net_device_ops qede_netdev_ops = {
 712         .ndo_open               = qede_open,
 713         .ndo_stop               = qede_close,
 714         .ndo_start_xmit         = qede_start_xmit,
 715         .ndo_select_queue       = qede_select_queue,
 716         .ndo_set_rx_mode        = qede_set_rx_mode,
 717         .ndo_set_mac_address    = qede_set_mac_addr,
 718         .ndo_validate_addr      = eth_validate_addr,
 719         .ndo_change_mtu         = qede_change_mtu,
 720         .ndo_eth_ioctl          = qede_ioctl,
 721         .ndo_tx_timeout         = qede_tx_timeout,
 722 #ifdef CONFIG_QED_SRIOV
 723         .ndo_set_vf_mac         = qede_set_vf_mac,
 724         .ndo_set_vf_vlan        = qede_set_vf_vlan,
 725         .ndo_set_vf_trust       = qede_set_vf_trust,
 726 #endif
 727         .ndo_vlan_rx_add_vid    = qede_vlan_rx_add_vid,
 728         .ndo_vlan_rx_kill_vid   = qede_vlan_rx_kill_vid,
 729         .ndo_fix_features       = qede_fix_features,
 730         .ndo_set_features       = qede_set_features,
 731         .ndo_get_stats64        = qede_get_stats64,
 732 #ifdef CONFIG_QED_SRIOV
 733         .ndo_set_vf_link_state  = qede_set_vf_link_state,
 734         .ndo_set_vf_spoofchk    = qede_set_vf_spoofchk,
 735         .ndo_get_vf_config      = qede_get_vf_config,
 736         .ndo_set_vf_rate        = qede_set_vf_rate,
 737 #endif
 738         .ndo_features_check     = qede_features_check,
 739         .ndo_bpf                = qede_xdp,
 740 #ifdef CONFIG_RFS_ACCEL
 741         .ndo_rx_flow_steer      = qede_rx_flow_steer,
 742 #endif
 743         .ndo_xdp_xmit           = qede_xdp_transmit,
 744         .ndo_setup_tc           = qede_setup_tc_offload,
 745 };
 746
 747 static const struct net_device_ops qede_netdev_vf_ops = {
 748         .ndo_open               = qede_open,
 749         .ndo_stop               = qede_close,
 750         .ndo_start_xmit         = qede_start_xmit,
 751         .ndo_select_queue       = qede_select_queue,
 752         .ndo_set_rx_mode        = qede_set_rx_mode,
 753         .ndo_set_mac_address    = qede_set_mac_addr,
 754         .ndo_validate_addr      = eth_validate_addr,
 755         .ndo_change_mtu         = qede_change_mtu,
 756         .ndo_vlan_rx_add_vid    = qede_vlan_rx_add_vid,
 757         .ndo_vlan_rx_kill_vid   = qede_vlan_rx_kill_vid,
 758         .ndo_fix_features       = qede_fix_features,
 759         .ndo_set_features       = qede_set_features,
 760         .ndo_get_stats64        = qede_get_stats64,
 761         .ndo_features_check     = qede_features_check,
 762 };
 763
 764 static const struct net_device_ops qede_netdev_vf_xdp_ops = {
 765         .ndo_open               = qede_open,
 766         .ndo_stop               = qede_close,
 767         .ndo_start_xmit         = qede_start_xmit,
 768         .ndo_select_queue       = qede_select_queue,
 769         .ndo_set_rx_mode        = qede_set_rx_mode,
 770         .ndo_set_mac_address    = qede_set_mac_addr,
 771         .ndo_validate_addr      = eth_validate_addr,
 772         .ndo_change_mtu         = qede_change_mtu,
 773         .ndo_vlan_rx_add_vid    = qede_vlan_rx_add_vid,
 774         .ndo_vlan_rx_kill_vid   = qede_vlan_rx_kill_vid,
 775         .ndo_fix_features       = qede_fix_features,
 776         .ndo_set_features       = qede_set_features,
 777         .ndo_get_stats64        = qede_get_stats64,
 778         .ndo_features_check     = qede_features_check,
 779         .ndo_bpf                = qede_xdp,
 780         .ndo_xdp_xmit           = qede_xdp_transmit,
 781 };
 782
 783 /* -------------------------------------------------------------------------
 784  * START OF PROBE / REMOVE
 785  * -------------------------------------------------------------------------
 786  */
 787
 788 static struct qede_dev *qede_alloc_etherdev(struct qed_dev *cdev,
 789                                             struct pci_dev *pdev,
 790                                             struct qed_dev_eth_info *info,
 791                                             u32 dp_module, u8 dp_level)
 792 {
 793         struct net_device *ndev;
 794         struct qede_dev *edev;
 795
 796         ndev = alloc_etherdev_mqs(sizeof(*edev),
 797                                   info->num_queues * info->num_tc,
 798                                   info->num_queues);
 799         if (!ndev) {
 800                 pr_err("etherdev allocation failed\n");
 801                 return NULL;
 802         }
 803
 804         edev = netdev_priv(ndev);
 805         edev->ndev = ndev;
 806         edev->cdev = cdev;
 807         edev->pdev = pdev;
 808         edev->dp_module = dp_module;
 809         edev->dp_level = dp_level;
 810         edev->ops = qed_ops;
 811
 812         if (is_kdump_kernel()) {
 813                 edev->q_num_rx_buffers = NUM_RX_BDS_KDUMP_MIN;
 814                 edev->q_num_tx_buffers = NUM_TX_BDS_KDUMP_MIN;
 815         } else {
 816                 edev->q_num_rx_buffers = NUM_RX_BDS_DEF;
 817                 edev->q_num_tx_buffers = NUM_TX_BDS_DEF;
 818         }
 819
 820         DP_INFO(edev, "Allocated netdev with %d tx queues and %d rx queues\n",
 821                 info->num_queues, info->num_queues);
 822
 823         SET_NETDEV_DEV(ndev, &pdev->dev);
 824
 825         memset(&edev->stats, 0, sizeof(edev->stats));
 826         memcpy(&edev->dev_info, info, sizeof(*info));
 827
 828         /* As ethtool doesn't have the ability to show WoL behavior as
 829          * 'default', if device supports it declare it's enabled.
 830          */
 831         if (edev->dev_info.common.wol_support)
 832                 edev->wol_enabled = true;
 833
 834         INIT_LIST_HEAD(&edev->vlan_list);
 835
 836         return edev;
 837 }
 838
 839 static void qede_init_ndev(struct qede_dev *edev)
 840 {
 841         struct net_device *ndev = edev->ndev;
 842         struct pci_dev *pdev = edev->pdev;
 843         bool udp_tunnel_enable = false;
 844         netdev_features_t hw_features;
 845
 846         pci_set_drvdata(pdev, ndev);
 847
 848         ndev->mem_start = edev->dev_info.common.pci_mem_start;
 849         ndev->base_addr = ndev->mem_start;
 850         ndev->mem_end = edev->dev_info.common.pci_mem_end;
 851         ndev->irq = edev->dev_info.common.pci_irq;
 852
 853         ndev->watchdog_timeo = TX_TIMEOUT;
 854
 855         if (IS_VF(edev)) {
 856                 if (edev->dev_info.xdp_supported)
 857                         ndev->netdev_ops = &qede_netdev_vf_xdp_ops;
 858                 else
 859                         ndev->netdev_ops = &qede_netdev_vf_ops;
 860         } else {
 861                 ndev->netdev_ops = &qede_netdev_ops;
 862         }
 863
 864         qede_set_ethtool_ops(ndev);
 865
 866         ndev->priv_flags |= IFF_UNICAST_FLT;
 867
 868         /* user-changeble features */
 869         hw_features = NETIF_F_GRO | NETIF_F_GRO_HW | NETIF_F_SG |
 870                       NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
 871                       NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_HW_TC;
 872
 873         if (edev->dev_info.common.b_arfs_capable)
 874                 hw_features |= NETIF_F_NTUPLE;
 875
 876         if (edev->dev_info.common.vxlan_enable ||
 877             edev->dev_info.common.geneve_enable)
 878                 udp_tunnel_enable = true;
 879
 880         if (udp_tunnel_enable || edev->dev_info.common.gre_enable) {
 881                 hw_features |= NETIF_F_TSO_ECN;
 882                 ndev->hw_enc_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
 883                                         NETIF_F_SG | NETIF_F_TSO |
 884                                         NETIF_F_TSO_ECN | NETIF_F_TSO6 |
 885                                         NETIF_F_RXCSUM;
 886         }
 887
 888         if (udp_tunnel_enable) {
 889                 hw_features |= (NETIF_F_GSO_UDP_TUNNEL |
 890                                 NETIF_F_GSO_UDP_TUNNEL_CSUM);
 891                 ndev->hw_enc_features |= (NETIF_F_GSO_UDP_TUNNEL |
 892                                           NETIF_F_GSO_UDP_TUNNEL_CSUM);
 893
 894                 qede_set_udp_tunnels(edev);
 895         }
 896
 897         if (edev->dev_info.common.gre_enable) {
 898                 hw_features |= (NETIF_F_GSO_GRE | NETIF_F_GSO_GRE_CSUM);
 899                 ndev->hw_enc_features |= (NETIF_F_GSO_GRE |
 900                                           NETIF_F_GSO_GRE_CSUM);
 901         }
 902
 903         ndev->vlan_features = hw_features | NETIF_F_RXHASH | NETIF_F_RXCSUM |
 904                               NETIF_F_HIGHDMA;
 905         ndev->features = hw_features | NETIF_F_RXHASH | NETIF_F_RXCSUM |
 906                          NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HIGHDMA |
 907                          NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_TX;
 908
 909         ndev->hw_features = hw_features;
 910
 911         ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
 912                              NETDEV_XDP_ACT_NDO_XMIT;
 913
 914         /* MTU range: 46 - 9600 */
 915         ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
 916         ndev->max_mtu = QEDE_MAX_JUMBO_PACKET_SIZE;
 917
 918         /* Set network device HW mac */
 919         eth_hw_addr_set(edev->ndev, edev->dev_info.common.hw_mac);
 920
 921         ndev->mtu = edev->dev_info.common.mtu;
 922 }
 923
 924 /* This function converts from 32b param to two params of level and module
 925  * Input 32b decoding:
 926  * b31 - enable all NOTICE prints. NOTICE prints are for deviation from the
 927  * 'happy' flow, e.g. memory allocation failed.
 928  * b30 - enable all INFO prints. INFO prints are for major steps in the flow
 929  * and provide important parameters.
 930  * b29-b0 - per-module bitmap, where each bit enables VERBOSE prints of that
 931  * module. VERBOSE prints are for tracking the specific flow in low level.
 932  *
 933  * Notice that the level should be that of the lowest required logs.
 934  */
 935 void qede_config_debug(uint debug, u32 *p_dp_module, u8 *p_dp_level)
 936 {
 937         *p_dp_level = QED_LEVEL_NOTICE;
 938         *p_dp_module = 0;
 939
 940         if (debug & QED_LOG_VERBOSE_MASK) {
 941                 *p_dp_level = QED_LEVEL_VERBOSE;
 942                 *p_dp_module = (debug & 0x3FFFFFFF);
 943         } else if (debug & QED_LOG_INFO_MASK) {
 944                 *p_dp_level = QED_LEVEL_INFO;
 945         } else if (debug & QED_LOG_NOTICE_MASK) {
 946                 *p_dp_level = QED_LEVEL_NOTICE;
 947         }
 948 }
 949
 950 static void qede_free_fp_array(struct qede_dev *edev)
 951 {
 952         if (edev->fp_array) {
 953                 struct qede_fastpath *fp;
 954                 int i;
 955
 956                 for_each_queue(i) {
 957                         fp = &edev->fp_array[i];
 958
 959                         kfree(fp->sb_info);
 960                         /* Handle mem alloc failure case where qede_init_fp
 961                          * didn't register xdp_rxq_info yet.
 962                          * Implicit only (fp->type & QEDE_FASTPATH_RX)
 963                          */
 964                         if (fp->rxq && xdp_rxq_info_is_reg(&fp->rxq->xdp_rxq))
 965                                 xdp_rxq_info_unreg(&fp->rxq->xdp_rxq);
 966                         kfree(fp->rxq);
 967                         kfree(fp->xdp_tx);
 968                         kfree(fp->txq);
 969                 }
 970                 kfree(edev->fp_array);
 971         }
 972
 973         edev->num_queues = 0;
 974         edev->fp_num_tx = 0;
 975         edev->fp_num_rx = 0;
 976 }
 977
 978 static int qede_alloc_fp_array(struct qede_dev *edev)
 979 {
 980         u8 fp_combined, fp_rx = edev->fp_num_rx;
 981         struct qede_fastpath *fp;
 982         int i;
 983
 984         edev->fp_array = kcalloc(QEDE_QUEUE_CNT(edev),
 985                                  sizeof(*edev->fp_array), GFP_KERNEL);
 986         if (!edev->fp_array) {
 987                 DP_NOTICE(edev, "fp array allocation failed\n");
 988                 goto err;
 989         }
 990
 991         if (!edev->coal_entry) {
 992                 edev->coal_entry = kcalloc(QEDE_MAX_RSS_CNT(edev),
 993                                            sizeof(*edev->coal_entry),
 994                                            GFP_KERNEL);
 995                 if (!edev->coal_entry) {
 996                         DP_ERR(edev, "coalesce entry allocation failed\n");
 997                         goto err;
 998                 }
 999         }
1000
1001         fp_combined = QEDE_QUEUE_CNT(edev) - fp_rx - edev->fp_num_tx;
1002
1003         /* Allocate the FP elements for Rx queues followed by combined and then
1004          * the Tx. This ordering should be maintained so that the respective
1005          * queues (Rx or Tx) will be together in the fastpath array and the
1006          * associated ids will be sequential.
1007          */
1008         for_each_queue(i) {
1009                 fp = &edev->fp_array[i];
1010
1011                 fp->sb_info = kzalloc(sizeof(*fp->sb_info), GFP_KERNEL);
1012                 if (!fp->sb_info) {
1013                         DP_NOTICE(edev, "sb info struct allocation failed\n");
1014                         goto err;
1015                 }
1016
1017                 if (fp_rx) {
1018                         fp->type = QEDE_FASTPATH_RX;
1019                         fp_rx--;
1020                 } else if (fp_combined) {
1021                         fp->type = QEDE_FASTPATH_COMBINED;
1022                         fp_combined--;
1023                 } else {
1024                         fp->type = QEDE_FASTPATH_TX;
1025                 }
1026
1027                 if (fp->type & QEDE_FASTPATH_TX) {
1028                         fp->txq = kcalloc(edev->dev_info.num_tc,
1029                                           sizeof(*fp->txq), GFP_KERNEL);
1030                         if (!fp->txq)
1031                                 goto err;
1032                 }
1033
1034                 if (fp->type & QEDE_FASTPATH_RX) {
1035                         fp->rxq = kzalloc(sizeof(*fp->rxq), GFP_KERNEL);
1036                         if (!fp->rxq)
1037                                 goto err;
1038
1039                         if (edev->xdp_prog) {
1040                                 fp->xdp_tx = kzalloc(sizeof(*fp->xdp_tx),
1041                                                      GFP_KERNEL);
1042                                 if (!fp->xdp_tx)
1043                                         goto err;
1044                                 fp->type |= QEDE_FASTPATH_XDP;
1045                         }
1046                 }
1047         }
1048
1049         return 0;
1050 err:
1051         qede_free_fp_array(edev);
1052         return -ENOMEM;
1053 }
1054
1055 /* The qede lock is used to protect driver state change and driver flows that
1056  * are not reentrant.
1057  */
1058 void __qede_lock(struct qede_dev *edev)
1059 {
1060         mutex_lock(&edev->qede_lock);
1061 }
1062
1063 void __qede_unlock(struct qede_dev *edev)
1064 {
1065         mutex_unlock(&edev->qede_lock);
1066 }
1067
1068 /* This version of the lock should be used when acquiring the RTNL lock is also
1069  * needed in addition to the internal qede lock.
1070  */
1071 static void qede_lock(struct qede_dev *edev)
1072 {
1073         rtnl_lock();
1074         __qede_lock(edev);
1075 }
1076
1077 static void qede_unlock(struct qede_dev *edev)
1078 {
1079         __qede_unlock(edev);
1080         rtnl_unlock();
1081 }
1082
1083 static void qede_periodic_task(struct work_struct *work)
1084 {
1085         struct qede_dev *edev = container_of(work, struct qede_dev,
1086                                              periodic_task.work);
1087
1088         qede_fill_by_demand_stats(edev);
1089         schedule_delayed_work(&edev->periodic_task, edev->stats_coal_ticks);
1090 }
1091
1092 static void qede_init_periodic_task(struct qede_dev *edev)
1093 {
1094         INIT_DELAYED_WORK(&edev->periodic_task, qede_periodic_task);
1095         spin_lock_init(&edev->stats_lock);
1096         edev->stats_coal_usecs = USEC_PER_SEC;
1097         edev->stats_coal_ticks = usecs_to_jiffies(USEC_PER_SEC);
1098 }
1099
1100 static void qede_sp_task(struct work_struct *work)
1101 {
1102         struct qede_dev *edev = container_of(work, struct qede_dev,
1103                                              sp_task.work);
1104
1105         /* Disable execution of this deferred work once
1106          * qede removal is in progress, this stop any future
1107          * scheduling of sp_task.
1108          */
1109         if (test_bit(QEDE_SP_DISABLE, &edev->sp_flags))
1110                 return;
1111
1112         /* The locking scheme depends on the specific flag:
1113          * In case of QEDE_SP_RECOVERY, acquiring the RTNL lock is required to
1114          * ensure that ongoing flows are ended and new ones are not started.
1115          * In other cases - only the internal qede lock should be acquired.
1116          */
1117
1118         if (test_and_clear_bit(QEDE_SP_RECOVERY, &edev->sp_flags)) {
1119                 cancel_delayed_work_sync(&edev->periodic_task);
1120 #ifdef CONFIG_QED_SRIOV
1121                 /* SRIOV must be disabled outside the lock to avoid a deadlock.
1122                  * The recovery of the active VFs is currently not supported.
1123                  */
1124                 if (pci_num_vf(edev->pdev))
1125                         qede_sriov_configure(edev->pdev, 0);
1126 #endif
1127                 qede_lock(edev);
1128                 qede_recovery_handler(edev);
1129                 qede_unlock(edev);
1130         }
1131
1132         __qede_lock(edev);
1133
1134         if (test_and_clear_bit(QEDE_SP_RX_MODE, &edev->sp_flags))
1135                 if (edev->state == QEDE_STATE_OPEN)
1136                         qede_config_rx_mode(edev->ndev);
1137
1138 #ifdef CONFIG_RFS_ACCEL
1139         if (test_and_clear_bit(QEDE_SP_ARFS_CONFIG, &edev->sp_flags)) {
1140                 if (edev->state == QEDE_STATE_OPEN)
1141                         qede_process_arfs_filters(edev, false);
1142         }
1143 #endif
1144         if (test_and_clear_bit(QEDE_SP_HW_ERR, &edev->sp_flags))
1145                 qede_generic_hw_err_handler(edev);
1146         __qede_unlock(edev);
1147
1148         if (test_and_clear_bit(QEDE_SP_AER, &edev->sp_flags)) {
1149 #ifdef CONFIG_QED_SRIOV
1150                 /* SRIOV must be disabled outside the lock to avoid a deadlock.
1151                  * The recovery of the active VFs is currently not supported.
1152                  */
1153                 if (pci_num_vf(edev->pdev))
1154                         qede_sriov_configure(edev->pdev, 0);
1155 #endif
1156                 edev->ops->common->recovery_process(edev->cdev);
1157         }
1158 }
1159
1160 static void qede_update_pf_params(struct qed_dev *cdev)
1161 {
1162         struct qed_pf_params pf_params;
1163         u16 num_cons;
1164
1165         /* 64 rx + 64 tx + 64 XDP */
1166         memset(&pf_params, 0, sizeof(struct qed_pf_params));
1167
1168         /* 1 rx + 1 xdp + max tx cos */
1169         num_cons = QED_MIN_L2_CONS;
1170
1171         pf_params.eth_pf_params.num_cons = (MAX_SB_PER_PF_MIMD - 1) * num_cons;
1172
1173         /* Same for VFs - make sure they'll have sufficient connections
1174          * to support XDP Tx queues.
1175          */
1176         pf_params.eth_pf_params.num_vf_cons = 48;
1177
1178         pf_params.eth_pf_params.num_arfs_filters = QEDE_RFS_MAX_FLTR;
1179         qed_ops->common->update_pf_params(cdev, &pf_params);
1180 }
1181
1182 #define QEDE_FW_VER_STR_SIZE    80
1183
1184 static void qede_log_probe(struct qede_dev *edev)
1185 {
1186         struct qed_dev_info *p_dev_info = &edev->dev_info.common;
1187         u8 buf[QEDE_FW_VER_STR_SIZE];
1188         size_t left_size;
1189
1190         snprintf(buf, QEDE_FW_VER_STR_SIZE,
1191                  "Storm FW %d.%d.%d.%d, Management FW %d.%d.%d.%d",
1192                  p_dev_info->fw_major, p_dev_info->fw_minor, p_dev_info->fw_rev,
1193                  p_dev_info->fw_eng,
1194                  (p_dev_info->mfw_rev & QED_MFW_VERSION_3_MASK) >>
1195                  QED_MFW_VERSION_3_OFFSET,
1196                  (p_dev_info->mfw_rev & QED_MFW_VERSION_2_MASK) >>
1197                  QED_MFW_VERSION_2_OFFSET,
1198                  (p_dev_info->mfw_rev & QED_MFW_VERSION_1_MASK) >>
1199                  QED_MFW_VERSION_1_OFFSET,
1200                  (p_dev_info->mfw_rev & QED_MFW_VERSION_0_MASK) >>
1201                  QED_MFW_VERSION_0_OFFSET);
1202
1203         left_size = QEDE_FW_VER_STR_SIZE - strlen(buf);
1204         if (p_dev_info->mbi_version && left_size)
1205                 snprintf(buf + strlen(buf), left_size,
1206                          " [MBI %d.%d.%d]",
1207                          (p_dev_info->mbi_version & QED_MBI_VERSION_2_MASK) >>
1208                          QED_MBI_VERSION_2_OFFSET,
1209                          (p_dev_info->mbi_version & QED_MBI_VERSION_1_MASK) >>
1210                          QED_MBI_VERSION_1_OFFSET,
1211                          (p_dev_info->mbi_version & QED_MBI_VERSION_0_MASK) >>
1212                          QED_MBI_VERSION_0_OFFSET);
1213
1214         pr_info("qede %02x:%02x.%02x: %s [%s]\n", edev->pdev->bus->number,
1215                 PCI_SLOT(edev->pdev->devfn), PCI_FUNC(edev->pdev->devfn),
1216                 buf, edev->ndev->name);
1217 }
1218
1219 enum qede_probe_mode {
1220         QEDE_PROBE_NORMAL,
1221         QEDE_PROBE_RECOVERY,
1222 };
1223
1224 static int __qede_probe(struct pci_dev *pdev, u32 dp_module, u8 dp_level,
1225                         bool is_vf, enum qede_probe_mode mode)
1226 {
1227         struct qed_probe_params probe_params;
1228         struct qed_slowpath_params sp_params;
1229         struct qed_dev_eth_info dev_info;
1230         struct qede_dev *edev;
1231         struct qed_dev *cdev;
1232         int rc;
1233
1234         if (unlikely(dp_level & QED_LEVEL_INFO))
1235                 pr_notice("Starting qede probe\n");
1236
1237         memset(&probe_params, 0, sizeof(probe_params));
1238         probe_params.protocol = QED_PROTOCOL_ETH;
1239         probe_params.dp_module = dp_module;
1240         probe_params.dp_level = dp_level;
1241         probe_params.is_vf = is_vf;
1242         probe_params.recov_in_prog = (mode == QEDE_PROBE_RECOVERY);
1243         cdev = qed_ops->common->probe(pdev, &probe_params);
1244         if (!cdev) {
1245                 rc = -ENODEV;
1246                 goto err0;
1247         }
1248
1249         qede_update_pf_params(cdev);
1250
1251         /* Start the Slowpath-process */
1252         memset(&sp_params, 0, sizeof(sp_params));
1253         sp_params.int_mode = QED_INT_MODE_MSIX;
1254         strscpy(sp_params.name, "qede LAN", QED_DRV_VER_STR_SIZE);
1255         rc = qed_ops->common->slowpath_start(cdev, &sp_params);
1256         if (rc) {
1257                 pr_notice("Cannot start slowpath\n");
1258                 goto err1;
1259         }
1260
1261         /* Learn information crucial for qede to progress */
1262         rc = qed_ops->fill_dev_info(cdev, &dev_info);
1263         if (rc)
1264                 goto err2;
1265
1266         if (mode != QEDE_PROBE_RECOVERY) {
1267                 edev = qede_alloc_etherdev(cdev, pdev, &dev_info, dp_module,
1268                                            dp_level);
1269                 if (!edev) {
1270                         rc = -ENOMEM;
1271                         goto err2;
1272                 }
1273
1274                 edev->devlink = qed_ops->common->devlink_register(cdev);
1275                 if (IS_ERR(edev->devlink)) {
1276                         DP_NOTICE(edev, "Cannot register devlink\n");
1277                         rc = PTR_ERR(edev->devlink);
1278                         edev->devlink = NULL;
1279                         goto err3;
1280                 }
1281         } else {
1282                 struct net_device *ndev = pci_get_drvdata(pdev);
1283                 struct qed_devlink *qdl;
1284
1285                 edev = netdev_priv(ndev);
1286                 qdl = devlink_priv(edev->devlink);
1287                 qdl->cdev = cdev;
1288                 edev->cdev = cdev;
1289                 memset(&edev->stats, 0, sizeof(edev->stats));
1290                 memcpy(&edev->dev_info, &dev_info, sizeof(dev_info));
1291         }
1292
1293         if (is_vf)
1294                 set_bit(QEDE_FLAGS_IS_VF, &edev->flags);
1295
1296         qede_init_ndev(edev);
1297
1298         rc = qede_rdma_dev_add(edev, (mode == QEDE_PROBE_RECOVERY));
1299         if (rc)
1300                 goto err3;
1301
1302         if (mode != QEDE_PROBE_RECOVERY) {
1303                 /* Prepare the lock prior to the registration of the netdev,
1304                  * as once it's registered we might reach flows requiring it
1305                  * [it's even possible to reach a flow needing it directly
1306                  * from there, although it's unlikely].
1307                  */
1308                 INIT_DELAYED_WORK(&edev->sp_task, qede_sp_task);
1309                 mutex_init(&edev->qede_lock);
1310                 qede_init_periodic_task(edev);
1311
1312                 rc = register_netdev(edev->ndev);
1313                 if (rc) {
1314                         DP_NOTICE(edev, "Cannot register net-device\n");
1315                         goto err4;
1316                 }
1317         }
1318
1319         edev->ops->common->set_name(cdev, edev->ndev->name);
1320
1321         /* PTP not supported on VFs */
1322         if (!is_vf)
1323                 qede_ptp_enable(edev);
1324
1325         edev->ops->register_ops(cdev, &qede_ll_ops, edev);
1326
1327 #ifdef CONFIG_DCB
1328         if (!IS_VF(edev))
1329                 qede_set_dcbnl_ops(edev->ndev);
1330 #endif
1331
1332         edev->rx_copybreak = QEDE_RX_HDR_SIZE;
1333
1334         qede_log_probe(edev);
1335
1336         /* retain user config (for example - after recovery) */
1337         if (edev->stats_coal_usecs)
1338                 schedule_delayed_work(&edev->periodic_task, 0);
1339
1340         return 0;
1341
1342 err4:
1343         qede_rdma_dev_remove(edev, (mode == QEDE_PROBE_RECOVERY));
1344 err3:
1345         if (mode != QEDE_PROBE_RECOVERY)
1346                 free_netdev(edev->ndev);
1347         else
1348                 edev->cdev = NULL;
1349 err2:
1350         qed_ops->common->slowpath_stop(cdev);
1351 err1:
1352         qed_ops->common->remove(cdev);
1353 err0:
1354         return rc;
1355 }
1356
1357 static int qede_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1358 {
1359         bool is_vf = false;
1360         u32 dp_module = 0;
1361         u8 dp_level = 0;
1362
1363         switch ((enum qede_pci_private)id->driver_data) {
1364         case QEDE_PRIVATE_VF:
1365                 if (debug & QED_LOG_VERBOSE_MASK)
1366                         dev_err(&pdev->dev, "Probing a VF\n");
1367                 is_vf = true;
1368                 break;
1369         default:
1370                 if (debug & QED_LOG_VERBOSE_MASK)
1371                         dev_err(&pdev->dev, "Probing a PF\n");
1372         }
1373
1374         qede_config_debug(debug, &dp_module, &dp_level);
1375
1376         return __qede_probe(pdev, dp_module, dp_level, is_vf,
1377                             QEDE_PROBE_NORMAL);
1378 }
1379
1380 enum qede_remove_mode {
1381         QEDE_REMOVE_NORMAL,
1382         QEDE_REMOVE_RECOVERY,
1383 };
1384
1385 static void __qede_remove(struct pci_dev *pdev, enum qede_remove_mode mode)
1386 {
1387         struct net_device *ndev = pci_get_drvdata(pdev);
1388         struct qede_dev *edev;
1389         struct qed_dev *cdev;
1390
1391         if (!ndev) {
1392                 dev_info(&pdev->dev, "Device has already been removed\n");
1393                 return;
1394         }
1395
1396         edev = netdev_priv(ndev);
1397         cdev = edev->cdev;
1398
1399         DP_INFO(edev, "Starting qede_remove\n");
1400
1401         qede_rdma_dev_remove(edev, (mode == QEDE_REMOVE_RECOVERY));
1402
1403         if (mode != QEDE_REMOVE_RECOVERY) {
1404                 set_bit(QEDE_SP_DISABLE, &edev->sp_flags);
1405                 unregister_netdev(ndev);
1406
1407                 cancel_delayed_work_sync(&edev->sp_task);
1408                 cancel_delayed_work_sync(&edev->periodic_task);
1409
1410                 edev->ops->common->set_power_state(cdev, PCI_D0);
1411
1412                 pci_set_drvdata(pdev, NULL);
1413         }
1414
1415         qede_ptp_disable(edev);
1416
1417         /* Use global ops since we've freed edev */
1418         qed_ops->common->slowpath_stop(cdev);
1419         if (system_state == SYSTEM_POWER_OFF)
1420                 return;
1421
1422         if (mode != QEDE_REMOVE_RECOVERY && edev->devlink) {
1423                 qed_ops->common->devlink_unregister(edev->devlink);
1424                 edev->devlink = NULL;
1425         }
1426         qed_ops->common->remove(cdev);
1427         edev->cdev = NULL;
1428
1429         /* Since this can happen out-of-sync with other flows,
1430          * don't release the netdevice until after slowpath stop
1431          * has been called to guarantee various other contexts
1432          * [e.g., QED register callbacks] won't break anything when
1433          * accessing the netdevice.
1434          */
1435         if (mode != QEDE_REMOVE_RECOVERY) {
1436                 kfree(edev->coal_entry);
1437                 free_netdev(ndev);
1438         }
1439
1440         dev_info(&pdev->dev, "Ending qede_remove successfully\n");
1441 }
1442
1443 static void qede_remove(struct pci_dev *pdev)
1444 {
1445         __qede_remove(pdev, QEDE_REMOVE_NORMAL);
1446 }
1447
1448 static void qede_shutdown(struct pci_dev *pdev)
1449 {
1450         __qede_remove(pdev, QEDE_REMOVE_NORMAL);
1451 }
1452
1453 /* -------------------------------------------------------------------------
1454  * START OF LOAD / UNLOAD
1455  * -------------------------------------------------------------------------
1456  */
1457
1458 static int qede_set_num_queues(struct qede_dev *edev)
1459 {
1460         int rc;
1461         u16 rss_num;
1462
1463         /* Setup queues according to possible resources*/
1464         if (edev->req_queues)
1465                 rss_num = edev->req_queues;
1466         else
1467                 rss_num = netif_get_num_default_rss_queues() *
1468                           edev->dev_info.common.num_hwfns;
1469
1470         rss_num = min_t(u16, QEDE_MAX_RSS_CNT(edev), rss_num);
1471
1472         rc = edev->ops->common->set_fp_int(edev->cdev, rss_num);
1473         if (rc > 0) {
1474                 /* Managed to request interrupts for our queues */
1475                 edev->num_queues = rc;
1476                 DP_INFO(edev, "Managed %d [of %d] RSS queues\n",
1477                         QEDE_QUEUE_CNT(edev), rss_num);
1478                 rc = 0;
1479         }
1480
1481         edev->fp_num_tx = edev->req_num_tx;
1482         edev->fp_num_rx = edev->req_num_rx;
1483
1484         return rc;
1485 }
1486
1487 static void qede_free_mem_sb(struct qede_dev *edev, struct qed_sb_info *sb_info,
1488                              u16 sb_id)
1489 {
1490         if (sb_info->sb_virt) {
1491                 edev->ops->common->sb_release(edev->cdev, sb_info, sb_id,
1492                                               QED_SB_TYPE_L2_QUEUE);
1493                 dma_free_coherent(&edev->pdev->dev, sizeof(*sb_info->sb_virt),
1494                                   (void *)sb_info->sb_virt, sb_info->sb_phys);
1495                 memset(sb_info, 0, sizeof(*sb_info));
1496         }
1497 }
1498
1499 /* This function allocates fast-path status block memory */
1500 static int qede_alloc_mem_sb(struct qede_dev *edev,
1501                              struct qed_sb_info *sb_info, u16 sb_id)
1502 {
1503         struct status_block *sb_virt;
1504         dma_addr_t sb_phys;
1505         int rc;
1506
1507         sb_virt = dma_alloc_coherent(&edev->pdev->dev,
1508                                      sizeof(*sb_virt), &sb_phys, GFP_KERNEL);
1509         if (!sb_virt) {
1510                 DP_ERR(edev, "Status block allocation failed\n");
1511                 return -ENOMEM;
1512         }
1513
1514         rc = edev->ops->common->sb_init(edev->cdev, sb_info,
1515                                         sb_virt, sb_phys, sb_id,
1516                                         QED_SB_TYPE_L2_QUEUE);
1517         if (rc) {
1518                 DP_ERR(edev, "Status block initialization failed\n");
1519                 dma_free_coherent(&edev->pdev->dev, sizeof(*sb_virt),
1520                                   sb_virt, sb_phys);
1521                 return rc;
1522         }
1523
1524         return 0;
1525 }
1526
1527 static void qede_free_rx_buffers(struct qede_dev *edev,
1528                                  struct qede_rx_queue *rxq)
1529 {
1530         u16 i;
1531
1532         for (i = rxq->sw_rx_cons; i != rxq->sw_rx_prod; i++) {
1533                 struct sw_rx_data *rx_buf;
1534                 struct page *data;
1535
1536                 rx_buf = &rxq->sw_rx_ring[i & NUM_RX_BDS_MAX];
1537                 data = rx_buf->data;
1538
1539                 dma_unmap_page(&edev->pdev->dev,
1540                                rx_buf->mapping, PAGE_SIZE, rxq->data_direction);
1541
1542                 rx_buf->data = NULL;
1543                 __free_page(data);
1544         }
1545 }
1546
1547 static void qede_free_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
1548 {
1549         /* Free rx buffers */
1550         qede_free_rx_buffers(edev, rxq);
1551
1552         /* Free the parallel SW ring */
1553         kfree(rxq->sw_rx_ring);
1554
1555         /* Free the real RQ ring used by FW */
1556         edev->ops->common->chain_free(edev->cdev, &rxq->rx_bd_ring);
1557         edev->ops->common->chain_free(edev->cdev, &rxq->rx_comp_ring);
1558 }
1559
1560 static void qede_set_tpa_param(struct qede_rx_queue *rxq)
1561 {
1562         int i;
1563
1564         for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) {
1565                 struct qede_agg_info *tpa_info = &rxq->tpa_info[i];
1566
1567                 tpa_info->state = QEDE_AGG_STATE_NONE;
1568         }
1569 }
1570
1571 /* This function allocates all memory needed per Rx queue */
1572 static int qede_alloc_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
1573 {
1574         struct qed_chain_init_params params = {
1575                 .cnt_type       = QED_CHAIN_CNT_TYPE_U16,
1576                 .num_elems      = RX_RING_SIZE,
1577         };
1578         struct qed_dev *cdev = edev->cdev;
1579         int i, rc, size;
1580
1581         rxq->num_rx_buffers = edev->q_num_rx_buffers;
1582
1583         rxq->rx_buf_size = NET_IP_ALIGN + ETH_OVERHEAD + edev->ndev->mtu;
1584
1585         rxq->rx_headroom = edev->xdp_prog ? XDP_PACKET_HEADROOM : NET_SKB_PAD;
1586         size = rxq->rx_headroom +
1587                SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1588
1589         /* Make sure that the headroom and  payload fit in a single page */
1590         if (rxq->rx_buf_size + size > PAGE_SIZE)
1591                 rxq->rx_buf_size = PAGE_SIZE - size;
1592
1593         /* Segment size to split a page in multiple equal parts,
1594          * unless XDP is used in which case we'd use the entire page.
1595          */
1596         if (!edev->xdp_prog) {
1597                 size = size + rxq->rx_buf_size;
1598                 rxq->rx_buf_seg_size = roundup_pow_of_two(size);
1599         } else {
1600                 rxq->rx_buf_seg_size = PAGE_SIZE;
1601                 edev->ndev->features &= ~NETIF_F_GRO_HW;
1602         }
1603
1604         /* Allocate the parallel driver ring for Rx buffers */
1605         size = sizeof(*rxq->sw_rx_ring) * RX_RING_SIZE;
1606         rxq->sw_rx_ring = kzalloc(size, GFP_KERNEL);
1607         if (!rxq->sw_rx_ring) {
1608                 DP_ERR(edev, "Rx buffers ring allocation failed\n");
1609                 rc = -ENOMEM;
1610                 goto err;
1611         }
1612
1613         /* Allocate FW Rx ring  */
1614         params.mode = QED_CHAIN_MODE_NEXT_PTR;
1615         params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
1616         params.elem_size = sizeof(struct eth_rx_bd);
1617
1618         rc = edev->ops->common->chain_alloc(cdev, &rxq->rx_bd_ring, &params);
1619         if (rc)
1620                 goto err;
1621
1622         /* Allocate FW completion ring */
1623         params.mode = QED_CHAIN_MODE_PBL;
1624         params.intended_use = QED_CHAIN_USE_TO_CONSUME;
1625         params.elem_size = sizeof(union eth_rx_cqe);
1626
1627         rc = edev->ops->common->chain_alloc(cdev, &rxq->rx_comp_ring, &params);
1628         if (rc)
1629                 goto err;
1630
1631         /* Allocate buffers for the Rx ring */
1632         rxq->filled_buffers = 0;
1633         for (i = 0; i < rxq->num_rx_buffers; i++) {
1634                 rc = qede_alloc_rx_buffer(rxq, false);
1635                 if (rc) {
1636                         DP_ERR(edev,
1637                                "Rx buffers allocation failed at index %d\n", i);
1638                         goto err;
1639                 }
1640         }
1641
1642         edev->gro_disable = !(edev->ndev->features & NETIF_F_GRO_HW);
1643         if (!edev->gro_disable)
1644                 qede_set_tpa_param(rxq);
1645 err:
1646         return rc;
1647 }
1648
1649 static void qede_free_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq)
1650 {
1651         /* Free the parallel SW ring */
1652         if (txq->is_xdp)
1653                 kfree(txq->sw_tx_ring.xdp);
1654         else
1655                 kfree(txq->sw_tx_ring.skbs);
1656
1657         /* Free the real RQ ring used by FW */
1658         edev->ops->common->chain_free(edev->cdev, &txq->tx_pbl);
1659 }
1660
1661 /* This function allocates all memory needed per Tx queue */
1662 static int qede_alloc_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq)
1663 {
1664         struct qed_chain_init_params params = {
1665                 .mode           = QED_CHAIN_MODE_PBL,
1666                 .intended_use   = QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1667                 .cnt_type       = QED_CHAIN_CNT_TYPE_U16,
1668                 .num_elems      = edev->q_num_tx_buffers,
1669                 .elem_size      = sizeof(union eth_tx_bd_types),
1670         };
1671         int size, rc;
1672
1673         txq->num_tx_buffers = edev->q_num_tx_buffers;
1674
1675         /* Allocate the parallel driver ring for Tx buffers */
1676         if (txq->is_xdp) {
1677                 size = sizeof(*txq->sw_tx_ring.xdp) * txq->num_tx_buffers;
1678                 txq->sw_tx_ring.xdp = kzalloc(size, GFP_KERNEL);
1679                 if (!txq->sw_tx_ring.xdp)
1680                         goto err;
1681         } else {
1682                 size = sizeof(*txq->sw_tx_ring.skbs) * txq->num_tx_buffers;
1683                 txq->sw_tx_ring.skbs = kzalloc(size, GFP_KERNEL);
1684                 if (!txq->sw_tx_ring.skbs)
1685                         goto err;
1686         }
1687
1688         rc = edev->ops->common->chain_alloc(edev->cdev, &txq->tx_pbl, &params);
1689         if (rc)
1690                 goto err;
1691
1692         return 0;
1693
1694 err:
1695         qede_free_mem_txq(edev, txq);
1696         return -ENOMEM;
1697 }
1698
1699 /* This function frees all memory of a single fp */
1700 static void qede_free_mem_fp(struct qede_dev *edev, struct qede_fastpath *fp)
1701 {
1702         qede_free_mem_sb(edev, fp->sb_info, fp->id);
1703
1704         if (fp->type & QEDE_FASTPATH_RX)
1705                 qede_free_mem_rxq(edev, fp->rxq);
1706
1707         if (fp->type & QEDE_FASTPATH_XDP)
1708                 qede_free_mem_txq(edev, fp->xdp_tx);
1709
1710         if (fp->type & QEDE_FASTPATH_TX) {
1711                 int cos;
1712
1713                 for_each_cos_in_txq(edev, cos)
1714                         qede_free_mem_txq(edev, &fp->txq[cos]);
1715         }
1716 }
1717
1718 /* This function allocates all memory needed for a single fp (i.e. an entity
1719  * which contains status block, one rx queue and/or multiple per-TC tx queues.
1720  */
1721 static int qede_alloc_mem_fp(struct qede_dev *edev, struct qede_fastpath *fp)
1722 {
1723         int rc = 0;
1724
1725         rc = qede_alloc_mem_sb(edev, fp->sb_info, fp->id);
1726         if (rc)
1727                 goto out;
1728
1729         if (fp->type & QEDE_FASTPATH_RX) {
1730                 rc = qede_alloc_mem_rxq(edev, fp->rxq);
1731                 if (rc)
1732                         goto out;
1733         }
1734
1735         if (fp->type & QEDE_FASTPATH_XDP) {
1736                 rc = qede_alloc_mem_txq(edev, fp->xdp_tx);
1737                 if (rc)
1738                         goto out;
1739         }
1740
1741         if (fp->type & QEDE_FASTPATH_TX) {
1742                 int cos;
1743
1744                 for_each_cos_in_txq(edev, cos) {
1745                         rc = qede_alloc_mem_txq(edev, &fp->txq[cos]);
1746                         if (rc)
1747                                 goto out;
1748                 }
1749         }
1750
1751 out:
1752         return rc;
1753 }
1754
1755 static void qede_free_mem_load(struct qede_dev *edev)
1756 {
1757         int i;
1758
1759         for_each_queue(i) {
1760                 struct qede_fastpath *fp = &edev->fp_array[i];
1761
1762                 qede_free_mem_fp(edev, fp);
1763         }
1764 }
1765
1766 /* This function allocates all qede memory at NIC load. */
1767 static int qede_alloc_mem_load(struct qede_dev *edev)
1768 {
1769         int rc = 0, queue_id;
1770
1771         for (queue_id = 0; queue_id < QEDE_QUEUE_CNT(edev); queue_id++) {
1772                 struct qede_fastpath *fp = &edev->fp_array[queue_id];
1773
1774                 rc = qede_alloc_mem_fp(edev, fp);
1775                 if (rc) {
1776                         DP_ERR(edev,
1777                                "Failed to allocate memory for fastpath - rss id = %d\n",
1778                                queue_id);
1779                         qede_free_mem_load(edev);
1780                         return rc;
1781                 }
1782         }
1783
1784         return 0;
1785 }
1786
1787 static void qede_empty_tx_queue(struct qede_dev *edev,
1788                                 struct qede_tx_queue *txq)
1789 {
1790         unsigned int pkts_compl = 0, bytes_compl = 0;
1791         struct netdev_queue *netdev_txq;
1792         int rc, len = 0;
1793
1794         netdev_txq = netdev_get_tx_queue(edev->ndev, txq->ndev_txq_id);
1795
1796         while (qed_chain_get_cons_idx(&txq->tx_pbl) !=
1797                qed_chain_get_prod_idx(&txq->tx_pbl)) {
1798                 DP_VERBOSE(edev, NETIF_MSG_IFDOWN,
1799                            "Freeing a packet on tx queue[%d]: chain_cons 0x%x, chain_prod 0x%x\n",
1800                            txq->index, qed_chain_get_cons_idx(&txq->tx_pbl),
1801                            qed_chain_get_prod_idx(&txq->tx_pbl));
1802
1803                 rc = qede_free_tx_pkt(edev, txq, &len);
1804                 if (rc) {
1805                         DP_NOTICE(edev,
1806                                   "Failed to free a packet on tx queue[%d]: chain_cons 0x%x, chain_prod 0x%x\n",
1807                                   txq->index,
1808                                   qed_chain_get_cons_idx(&txq->tx_pbl),
1809                                   qed_chain_get_prod_idx(&txq->tx_pbl));
1810                         break;
1811                 }
1812
1813                 bytes_compl += len;
1814                 pkts_compl++;
1815                 txq->sw_tx_cons++;
1816         }
1817
1818         netdev_tx_completed_queue(netdev_txq, pkts_compl, bytes_compl);
1819 }
1820
1821 static void qede_empty_tx_queues(struct qede_dev *edev)
1822 {
1823         int i;
1824
1825         for_each_queue(i)
1826                 if (edev->fp_array[i].type & QEDE_FASTPATH_TX) {
1827                         int cos;
1828
1829                         for_each_cos_in_txq(edev, cos) {
1830                                 struct qede_fastpath *fp;
1831
1832                                 fp = &edev->fp_array[i];
1833                                 qede_empty_tx_queue(edev,
1834                                                     &fp->txq[cos]);
1835                         }
1836                 }
1837 }
1838
1839 /* This function inits fp content and resets the SB, RXQ and TXQ structures */
1840 static void qede_init_fp(struct qede_dev *edev)
1841 {
1842         int queue_id, rxq_index = 0, txq_index = 0;
1843         struct qede_fastpath *fp;
1844         bool init_xdp = false;
1845
1846         for_each_queue(queue_id) {
1847                 fp = &edev->fp_array[queue_id];
1848
1849                 fp->edev = edev;
1850                 fp->id = queue_id;
1851
1852                 if (fp->type & QEDE_FASTPATH_XDP) {
1853                         fp->xdp_tx->index = QEDE_TXQ_IDX_TO_XDP(edev,
1854                                                                 rxq_index);
1855                         fp->xdp_tx->is_xdp = 1;
1856
1857                         spin_lock_init(&fp->xdp_tx->xdp_tx_lock);
1858                         init_xdp = true;
1859                 }
1860
1861                 if (fp->type & QEDE_FASTPATH_RX) {
1862                         fp->rxq->rxq_id = rxq_index++;
1863
1864                         /* Determine how to map buffers for this queue */
1865                         if (fp->type & QEDE_FASTPATH_XDP)
1866                                 fp->rxq->data_direction = DMA_BIDIRECTIONAL;
1867                         else
1868                                 fp->rxq->data_direction = DMA_FROM_DEVICE;
1869                         fp->rxq->dev = &edev->pdev->dev;
1870
1871                         /* Driver have no error path from here */
1872                         WARN_ON(xdp_rxq_info_reg(&fp->rxq->xdp_rxq, edev->ndev,
1873                                                  fp->rxq->rxq_id, 0) < 0);
1874
1875                         if (xdp_rxq_info_reg_mem_model(&fp->rxq->xdp_rxq,
1876                                                        MEM_TYPE_PAGE_ORDER0,
1877                                                        NULL)) {
1878                                 DP_NOTICE(edev,
1879                                           "Failed to register XDP memory model\n");
1880                         }
1881                 }
1882
1883                 if (fp->type & QEDE_FASTPATH_TX) {
1884                         int cos;
1885
1886                         for_each_cos_in_txq(edev, cos) {
1887                                 struct qede_tx_queue *txq = &fp->txq[cos];
1888                                 u16 ndev_tx_id;
1889
1890                                 txq->cos = cos;
1891                                 txq->index = txq_index;
1892                                 ndev_tx_id = QEDE_TXQ_TO_NDEV_TXQ_ID(edev, txq);
1893                                 txq->ndev_txq_id = ndev_tx_id;
1894
1895                                 if (edev->dev_info.is_legacy)
1896                                         txq->is_legacy = true;
1897                                 txq->dev = &edev->pdev->dev;
1898                         }
1899
1900                         txq_index++;
1901                 }
1902
1903                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1904                          edev->ndev->name, queue_id);
1905         }
1906
1907         if (init_xdp) {
1908                 edev->total_xdp_queues = QEDE_RSS_COUNT(edev);
1909                 DP_INFO(edev, "Total XDP queues: %u\n", edev->total_xdp_queues);
1910         }
1911 }
1912
1913 static int qede_set_real_num_queues(struct qede_dev *edev)
1914 {
1915         int rc = 0;
1916
1917         rc = netif_set_real_num_tx_queues(edev->ndev,
1918                                           QEDE_TSS_COUNT(edev) *
1919                                           edev->dev_info.num_tc);
1920         if (rc) {
1921                 DP_NOTICE(edev, "Failed to set real number of Tx queues\n");
1922                 return rc;
1923         }
1924
1925         rc = netif_set_real_num_rx_queues(edev->ndev, QEDE_RSS_COUNT(edev));
1926         if (rc) {
1927                 DP_NOTICE(edev, "Failed to set real number of Rx queues\n");
1928                 return rc;
1929         }
1930
1931         return 0;
1932 }
1933
1934 static void qede_napi_disable_remove(struct qede_dev *edev)
1935 {
1936         int i;
1937
1938         for_each_queue(i) {
1939                 napi_disable(&edev->fp_array[i].napi);
1940
1941                 netif_napi_del(&edev->fp_array[i].napi);
1942         }
1943 }
1944
1945 static void qede_napi_add_enable(struct qede_dev *edev)
1946 {
1947         int i;
1948
1949         /* Add NAPI objects */
1950         for_each_queue(i) {
1951                 netif_napi_add(edev->ndev, &edev->fp_array[i].napi, qede_poll);
1952                 napi_enable(&edev->fp_array[i].napi);
1953         }
1954 }
1955
1956 static void qede_sync_free_irqs(struct qede_dev *edev)
1957 {
1958         int i;
1959
1960         for (i = 0; i < edev->int_info.used_cnt; i++) {
1961                 if (edev->int_info.msix_cnt) {
1962                         free_irq(edev->int_info.msix[i].vector,
1963                                  &edev->fp_array[i]);
1964                 } else {
1965                         edev->ops->common->simd_handler_clean(edev->cdev, i);
1966                 }
1967         }
1968
1969         edev->int_info.used_cnt = 0;
1970         edev->int_info.msix_cnt = 0;
1971 }
1972
1973 static int qede_req_msix_irqs(struct qede_dev *edev)
1974 {
1975         int i, rc;
1976
1977         /* Sanitize number of interrupts == number of prepared RSS queues */
1978         if (QEDE_QUEUE_CNT(edev) > edev->int_info.msix_cnt) {
1979                 DP_ERR(edev,
1980                        "Interrupt mismatch: %d RSS queues > %d MSI-x vectors\n",
1981                        QEDE_QUEUE_CNT(edev), edev->int_info.msix_cnt);
1982                 return -EINVAL;
1983         }
1984
1985         for (i = 0; i < QEDE_QUEUE_CNT(edev); i++) {
1986 #ifdef CONFIG_RFS_ACCEL
1987                 struct qede_fastpath *fp = &edev->fp_array[i];
1988
1989                 if (edev->ndev->rx_cpu_rmap && (fp->type & QEDE_FASTPATH_RX)) {
1990                         rc = irq_cpu_rmap_add(edev->ndev->rx_cpu_rmap,
1991                                               edev->int_info.msix[i].vector);
1992                         if (rc) {
1993                                 DP_ERR(edev, "Failed to add CPU rmap\n");
1994                                 qede_free_arfs(edev);
1995                         }
1996                 }
1997 #endif
1998                 rc = request_irq(edev->int_info.msix[i].vector,
1999                                  qede_msix_fp_int, 0, edev->fp_array[i].name,
2000                                  &edev->fp_array[i]);
2001                 if (rc) {
2002                         DP_ERR(edev, "Request fp %d irq failed\n", i);
2003 #ifdef CONFIG_RFS_ACCEL
2004                         if (edev->ndev->rx_cpu_rmap)
2005                                 free_irq_cpu_rmap(edev->ndev->rx_cpu_rmap);
2006
2007                         edev->ndev->rx_cpu_rmap = NULL;
2008 #endif
2009                         qede_sync_free_irqs(edev);
2010                         return rc;
2011                 }
2012                 DP_VERBOSE(edev, NETIF_MSG_INTR,
2013                            "Requested fp irq for %s [entry %d]. Cookie is at %p\n",
2014                            edev->fp_array[i].name, i,
2015                            &edev->fp_array[i]);
2016                 edev->int_info.used_cnt++;
2017         }
2018
2019         return 0;
2020 }
2021
2022 static void qede_simd_fp_handler(void *cookie)
2023 {
2024         struct qede_fastpath *fp = (struct qede_fastpath *)cookie;
2025
2026         napi_schedule_irqoff(&fp->napi);
2027 }
2028
2029 static int qede_setup_irqs(struct qede_dev *edev)
2030 {
2031         int i, rc = 0;
2032
2033         /* Learn Interrupt configuration */
2034         rc = edev->ops->common->get_fp_int(edev->cdev, &edev->int_info);
2035         if (rc)
2036                 return rc;
2037
2038         if (edev->int_info.msix_cnt) {
2039                 rc = qede_req_msix_irqs(edev);
2040                 if (rc)
2041                         return rc;
2042                 edev->ndev->irq = edev->int_info.msix[0].vector;
2043         } else {
2044                 const struct qed_common_ops *ops;
2045
2046                 /* qed should learn receive the RSS ids and callbacks */
2047                 ops = edev->ops->common;
2048                 for (i = 0; i < QEDE_QUEUE_CNT(edev); i++)
2049                         ops->simd_handler_config(edev->cdev,
2050                                                  &edev->fp_array[i], i,
2051                                                  qede_simd_fp_handler);
2052                 edev->int_info.used_cnt = QEDE_QUEUE_CNT(edev);
2053         }
2054         return 0;
2055 }
2056
2057 static int qede_drain_txq(struct qede_dev *edev,
2058                           struct qede_tx_queue *txq, bool allow_drain)
2059 {
2060         int rc, cnt = 1000;
2061
2062         while (txq->sw_tx_cons != txq->sw_tx_prod) {
2063                 if (!cnt) {
2064                         if (allow_drain) {
2065                                 DP_NOTICE(edev,
2066                                           "Tx queue[%d] is stuck, requesting MCP to drain\n",
2067                                           txq->index);
2068                                 rc = edev->ops->common->drain(edev->cdev);
2069                                 if (rc)
2070                                         return rc;
2071                                 return qede_drain_txq(edev, txq, false);
2072                         }
2073                         DP_NOTICE(edev,
2074                                   "Timeout waiting for tx queue[%d]: PROD=%d, CONS=%d\n",
2075                                   txq->index, txq->sw_tx_prod,
2076                                   txq->sw_tx_cons);
2077                         return -ENODEV;
2078                 }
2079                 cnt--;
2080                 usleep_range(1000, 2000);
2081                 barrier();
2082         }
2083
2084         /* FW finished processing, wait for HW to transmit all tx packets */
2085         usleep_range(1000, 2000);
2086
2087         return 0;
2088 }
2089
2090 static int qede_stop_txq(struct qede_dev *edev,
2091                          struct qede_tx_queue *txq, int rss_id)
2092 {
2093         /* delete doorbell from doorbell recovery mechanism */
2094         edev->ops->common->db_recovery_del(edev->cdev, txq->doorbell_addr,
2095                                            &txq->tx_db);
2096
2097         return edev->ops->q_tx_stop(edev->cdev, rss_id, txq->handle);
2098 }
2099
2100 static int qede_stop_queues(struct qede_dev *edev)
2101 {
2102         struct qed_update_vport_params *vport_update_params;
2103         struct qed_dev *cdev = edev->cdev;
2104         struct qede_fastpath *fp;
2105         int rc, i;
2106
2107         /* Disable the vport */
2108         vport_update_params = vzalloc(sizeof(*vport_update_params));
2109         if (!vport_update_params)
2110                 return -ENOMEM;
2111
2112         vport_update_params->vport_id = 0;
2113         vport_update_params->update_vport_active_flg = 1;
2114         vport_update_params->vport_active_flg = 0;
2115         vport_update_params->update_rss_flg = 0;
2116
2117         rc = edev->ops->vport_update(cdev, vport_update_params);
2118         vfree(vport_update_params);
2119
2120         if (rc) {
2121                 DP_ERR(edev, "Failed to update vport\n");
2122                 return rc;
2123         }
2124
2125         /* Flush Tx queues. If needed, request drain from MCP */
2126         for_each_queue(i) {
2127                 fp = &edev->fp_array[i];
2128
2129                 if (fp->type & QEDE_FASTPATH_TX) {
2130                         int cos;
2131
2132                         for_each_cos_in_txq(edev, cos) {
2133                                 rc = qede_drain_txq(edev, &fp->txq[cos], true);
2134                                 if (rc)
2135                                         return rc;
2136                         }
2137                 }
2138
2139                 if (fp->type & QEDE_FASTPATH_XDP) {
2140                         rc = qede_drain_txq(edev, fp->xdp_tx, true);
2141                         if (rc)
2142                                 return rc;
2143                 }
2144         }
2145
2146         /* Stop all Queues in reverse order */
2147         for (i = QEDE_QUEUE_CNT(edev) - 1; i >= 0; i--) {
2148                 fp = &edev->fp_array[i];
2149
2150                 /* Stop the Tx Queue(s) */
2151                 if (fp->type & QEDE_FASTPATH_TX) {
2152                         int cos;
2153
2154                         for_each_cos_in_txq(edev, cos) {
2155                                 rc = qede_stop_txq(edev, &fp->txq[cos], i);
2156                                 if (rc)
2157                                         return rc;
2158                         }
2159                 }
2160
2161                 /* Stop the Rx Queue */
2162                 if (fp->type & QEDE_FASTPATH_RX) {
2163                         rc = edev->ops->q_rx_stop(cdev, i, fp->rxq->handle);
2164                         if (rc) {
2165                                 DP_ERR(edev, "Failed to stop RXQ #%d\n", i);
2166                                 return rc;
2167                         }
2168                 }
2169
2170                 /* Stop the XDP forwarding queue */
2171                 if (fp->type & QEDE_FASTPATH_XDP) {
2172                         rc = qede_stop_txq(edev, fp->xdp_tx, i);
2173                         if (rc)
2174                                 return rc;
2175
2176                         bpf_prog_put(fp->rxq->xdp_prog);
2177                 }
2178         }
2179
2180         /* Stop the vport */
2181         rc = edev->ops->vport_stop(cdev, 0);
2182         if (rc)
2183                 DP_ERR(edev, "Failed to stop VPORT\n");
2184
2185         return rc;
2186 }
2187
2188 static int qede_start_txq(struct qede_dev *edev,
2189                           struct qede_fastpath *fp,
2190                           struct qede_tx_queue *txq, u8 rss_id, u16 sb_idx)
2191 {
2192         dma_addr_t phys_table = qed_chain_get_pbl_phys(&txq->tx_pbl);
2193         u32 page_cnt = qed_chain_get_page_cnt(&txq->tx_pbl);
2194         struct qed_queue_start_common_params params;
2195         struct qed_txq_start_ret_params ret_params;
2196         int rc;
2197
2198         memset(&params, 0, sizeof(params));
2199         memset(&ret_params, 0, sizeof(ret_params));
2200
2201         /* Let the XDP queue share the queue-zone with one of the regular txq.
2202          * We don't really care about its coalescing.
2203          */
2204         if (txq->is_xdp)
2205                 params.queue_id = QEDE_TXQ_XDP_TO_IDX(edev, txq);
2206         else
2207                 params.queue_id = txq->index;
2208
2209         params.p_sb = fp->sb_info;
2210         params.sb_idx = sb_idx;
2211         params.tc = txq->cos;
2212
2213         rc = edev->ops->q_tx_start(edev->cdev, rss_id, &params, phys_table,
2214                                    page_cnt, &ret_params);
2215         if (rc) {
2216                 DP_ERR(edev, "Start TXQ #%d failed %d\n", txq->index, rc);
2217                 return rc;
2218         }
2219
2220         txq->doorbell_addr = ret_params.p_doorbell;
2221         txq->handle = ret_params.p_handle;
2222
2223         /* Determine the FW consumer address associated */
2224         txq->hw_cons_ptr = &fp->sb_info->sb_virt->pi_array[sb_idx];
2225
2226         /* Prepare the doorbell parameters */
2227         SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_DEST, DB_DEST_XCM);
2228         SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD, DB_AGG_CMD_SET);
2229         SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_VAL_SEL,
2230                   DQ_XCM_ETH_TX_BD_PROD_CMD);
2231         txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD;
2232
2233         /* register doorbell with doorbell recovery mechanism */
2234         rc = edev->ops->common->db_recovery_add(edev->cdev, txq->doorbell_addr,
2235                                                 &txq->tx_db, DB_REC_WIDTH_32B,
2236                                                 DB_REC_KERNEL);
2237
2238         return rc;
2239 }
2240
2241 static int qede_start_queues(struct qede_dev *edev, bool clear_stats)
2242 {
2243         int vlan_removal_en = 1;
2244         struct qed_dev *cdev = edev->cdev;
2245         struct qed_dev_info *qed_info = &edev->dev_info.common;
2246         struct qed_update_vport_params *vport_update_params;
2247         struct qed_queue_start_common_params q_params;
2248         struct qed_start_vport_params start = {0};
2249         int rc, i;
2250
2251         if (!edev->num_queues) {
2252                 DP_ERR(edev,
2253                        "Cannot update V-VPORT as active as there are no Rx queues\n");
2254                 return -EINVAL;
2255         }
2256
2257         vport_update_params = vzalloc(sizeof(*vport_update_params));
2258         if (!vport_update_params)
2259                 return -ENOMEM;
2260
2261         start.handle_ptp_pkts = !!(edev->ptp);
2262         start.gro_enable = !edev->gro_disable;
2263         start.mtu = edev->ndev->mtu;
2264         start.vport_id = 0;
2265         start.drop_ttl0 = true;
2266         start.remove_inner_vlan = vlan_removal_en;
2267         start.clear_stats = clear_stats;
2268
2269         rc = edev->ops->vport_start(cdev, &start);
2270
2271         if (rc) {
2272                 DP_ERR(edev, "Start V-PORT failed %d\n", rc);
2273                 goto out;
2274         }
2275
2276         DP_VERBOSE(edev, NETIF_MSG_IFUP,
2277                    "Start vport ramrod passed, vport_id = %d, MTU = %d, vlan_removal_en = %d\n",
2278                    start.vport_id, edev->ndev->mtu + 0xe, vlan_removal_en);
2279
2280         for_each_queue(i) {
2281                 struct qede_fastpath *fp = &edev->fp_array[i];
2282                 dma_addr_t p_phys_table;
2283                 u32 page_cnt;
2284
2285                 if (fp->type & QEDE_FASTPATH_RX) {
2286                         struct qed_rxq_start_ret_params ret_params;
2287                         struct qede_rx_queue *rxq = fp->rxq;
2288                         __le16 *val;
2289
2290                         memset(&ret_params, 0, sizeof(ret_params));
2291                         memset(&q_params, 0, sizeof(q_params));
2292                         q_params.queue_id = rxq->rxq_id;
2293                         q_params.vport_id = 0;
2294                         q_params.p_sb = fp->sb_info;
2295                         q_params.sb_idx = RX_PI;
2296
2297                         p_phys_table =
2298                             qed_chain_get_pbl_phys(&rxq->rx_comp_ring);
2299                         page_cnt = qed_chain_get_page_cnt(&rxq->rx_comp_ring);
2300
2301                         rc = edev->ops->q_rx_start(cdev, i, &q_params,
2302                                                    rxq->rx_buf_size,
2303                                                    rxq->rx_bd_ring.p_phys_addr,
2304                                                    p_phys_table,
2305                                                    page_cnt, &ret_params);
2306                         if (rc) {
2307                                 DP_ERR(edev, "Start RXQ #%d failed %d\n", i,
2308                                        rc);
2309                                 goto out;
2310                         }
2311
2312                         /* Use the return parameters */
2313                         rxq->hw_rxq_prod_addr = ret_params.p_prod;
2314                         rxq->handle = ret_params.p_handle;
2315
2316                         val = &fp->sb_info->sb_virt->pi_array[RX_PI];
2317                         rxq->hw_cons_ptr = val;
2318
2319                         qede_update_rx_prod(edev, rxq);
2320                 }
2321
2322                 if (fp->type & QEDE_FASTPATH_XDP) {
2323                         rc = qede_start_txq(edev, fp, fp->xdp_tx, i, XDP_PI);
2324                         if (rc)
2325                                 goto out;
2326
2327                         bpf_prog_add(edev->xdp_prog, 1);
2328                         fp->rxq->xdp_prog = edev->xdp_prog;
2329                 }
2330
2331                 if (fp->type & QEDE_FASTPATH_TX) {
2332                         int cos;
2333
2334                         for_each_cos_in_txq(edev, cos) {
2335                                 rc = qede_start_txq(edev, fp, &fp->txq[cos], i,
2336                                                     TX_PI(cos));
2337                                 if (rc)
2338                                         goto out;
2339                         }
2340                 }
2341         }
2342
2343         /* Prepare and send the vport enable */
2344         vport_update_params->vport_id = start.vport_id;
2345         vport_update_params->update_vport_active_flg = 1;
2346         vport_update_params->vport_active_flg = 1;
2347
2348         if ((qed_info->b_inter_pf_switch || pci_num_vf(edev->pdev)) &&
2349             qed_info->tx_switching) {
2350                 vport_update_params->update_tx_switching_flg = 1;
2351                 vport_update_params->tx_switching_flg = 1;
2352         }
2353
2354         qede_fill_rss_params(edev, &vport_update_params->rss_params,
2355                              &vport_update_params->update_rss_flg);
2356
2357         rc = edev->ops->vport_update(cdev, vport_update_params);
2358         if (rc)
2359                 DP_ERR(edev, "Update V-PORT failed %d\n", rc);
2360
2361 out:
2362         vfree(vport_update_params);
2363         return rc;
2364 }
2365
2366 enum qede_unload_mode {
2367         QEDE_UNLOAD_NORMAL,
2368         QEDE_UNLOAD_RECOVERY,
2369 };
2370
2371 static void qede_unload(struct qede_dev *edev, enum qede_unload_mode mode,
2372                         bool is_locked)
2373 {
2374         struct qed_link_params link_params;
2375         int rc;
2376
2377         DP_INFO(edev, "Starting qede unload\n");
2378
2379         if (!is_locked)
2380                 __qede_lock(edev);
2381
2382         clear_bit(QEDE_FLAGS_LINK_REQUESTED, &edev->flags);
2383
2384         if (mode != QEDE_UNLOAD_RECOVERY)
2385                 edev->state = QEDE_STATE_CLOSED;
2386
2387         qede_rdma_dev_event_close(edev);
2388
2389         /* Close OS Tx */
2390         netif_tx_disable(edev->ndev);
2391         netif_carrier_off(edev->ndev);
2392
2393         if (mode != QEDE_UNLOAD_RECOVERY) {
2394                 /* Reset the link */
2395                 memset(&link_params, 0, sizeof(link_params));
2396                 link_params.link_up = false;
2397                 edev->ops->common->set_link(edev->cdev, &link_params);
2398
2399                 rc = qede_stop_queues(edev);
2400                 if (rc) {
2401 #ifdef CONFIG_RFS_ACCEL
2402                         if (edev->dev_info.common.b_arfs_capable) {
2403                                 qede_poll_for_freeing_arfs_filters(edev);
2404                                 if (edev->ndev->rx_cpu_rmap)
2405                                         free_irq_cpu_rmap(edev->ndev->rx_cpu_rmap);
2406
2407                                 edev->ndev->rx_cpu_rmap = NULL;
2408                         }
2409 #endif
2410                         qede_sync_free_irqs(edev);
2411                         goto out;
2412                 }
2413
2414                 DP_INFO(edev, "Stopped Queues\n");
2415         }
2416
2417         qede_vlan_mark_nonconfigured(edev);
2418         edev->ops->fastpath_stop(edev->cdev);
2419
2420         if (edev->dev_info.common.b_arfs_capable) {
2421                 qede_poll_for_freeing_arfs_filters(edev);
2422                 qede_free_arfs(edev);
2423         }
2424
2425         /* Release the interrupts */
2426         qede_sync_free_irqs(edev);
2427         edev->ops->common->set_fp_int(edev->cdev, 0);
2428
2429         qede_napi_disable_remove(edev);
2430
2431         if (mode == QEDE_UNLOAD_RECOVERY)
2432                 qede_empty_tx_queues(edev);
2433
2434         qede_free_mem_load(edev);
2435         qede_free_fp_array(edev);
2436
2437 out:
2438         if (!is_locked)
2439                 __qede_unlock(edev);
2440
2441         if (mode != QEDE_UNLOAD_RECOVERY)
2442                 DP_NOTICE(edev, "Link is down\n");
2443
2444         edev->ptp_skip_txts = 0;
2445
2446         DP_INFO(edev, "Ending qede unload\n");
2447 }
2448
2449 enum qede_load_mode {
2450         QEDE_LOAD_NORMAL,
2451         QEDE_LOAD_RELOAD,
2452         QEDE_LOAD_RECOVERY,
2453 };
2454
2455 static int qede_load(struct qede_dev *edev, enum qede_load_mode mode,
2456                      bool is_locked)
2457 {
2458         struct qed_link_params link_params;
2459         struct ethtool_coalesce coal = {};
2460         u8 num_tc;
2461         int rc, i;
2462
2463         DP_INFO(edev, "Starting qede load\n");
2464
2465         if (!is_locked)
2466                 __qede_lock(edev);
2467
2468         rc = qede_set_num_queues(edev);
2469         if (rc)
2470                 goto out;
2471
2472         rc = qede_alloc_fp_array(edev);
2473         if (rc)
2474                 goto out;
2475
2476         qede_init_fp(edev);
2477
2478         rc = qede_alloc_mem_load(edev);
2479         if (rc)
2480                 goto err1;
2481         DP_INFO(edev, "Allocated %d Rx, %d Tx queues\n",
2482                 QEDE_RSS_COUNT(edev), QEDE_TSS_COUNT(edev));
2483
2484         rc = qede_set_real_num_queues(edev);
2485         if (rc)
2486                 goto err2;
2487
2488         if (qede_alloc_arfs(edev)) {
2489                 edev->ndev->features &= ~NETIF_F_NTUPLE;
2490                 edev->dev_info.common.b_arfs_capable = false;
2491         }
2492
2493         qede_napi_add_enable(edev);
2494         DP_INFO(edev, "Napi added and enabled\n");
2495
2496         rc = qede_setup_irqs(edev);
2497         if (rc)
2498                 goto err3;
2499         DP_INFO(edev, "Setup IRQs succeeded\n");
2500
2501         rc = qede_start_queues(edev, mode != QEDE_LOAD_RELOAD);
2502         if (rc)
2503                 goto err4;
2504         DP_INFO(edev, "Start VPORT, RXQ and TXQ succeeded\n");
2505
2506         num_tc = netdev_get_num_tc(edev->ndev);
2507         num_tc = num_tc ? num_tc : edev->dev_info.num_tc;
2508         qede_setup_tc(edev->ndev, num_tc);
2509
2510         /* Program un-configured VLANs */
2511         qede_configure_vlan_filters(edev);
2512
2513         set_bit(QEDE_FLAGS_LINK_REQUESTED, &edev->flags);
2514
2515         /* Ask for link-up using current configuration */
2516         memset(&link_params, 0, sizeof(link_params));
2517         link_params.link_up = true;
2518         edev->ops->common->set_link(edev->cdev, &link_params);
2519
2520         edev->state = QEDE_STATE_OPEN;
2521
2522         coal.rx_coalesce_usecs = QED_DEFAULT_RX_USECS;
2523         coal.tx_coalesce_usecs = QED_DEFAULT_TX_USECS;
2524
2525         for_each_queue(i) {
2526                 if (edev->coal_entry[i].isvalid) {
2527                         coal.rx_coalesce_usecs = edev->coal_entry[i].rxc;
2528                         coal.tx_coalesce_usecs = edev->coal_entry[i].txc;
2529                 }
2530                 __qede_unlock(edev);
2531                 qede_set_per_coalesce(edev->ndev, i, &coal);
2532                 __qede_lock(edev);
2533         }
2534         DP_INFO(edev, "Ending successfully qede load\n");
2535
2536         goto out;
2537 err4:
2538         qede_sync_free_irqs(edev);
2539 err3:
2540         qede_napi_disable_remove(edev);
2541 err2:
2542         qede_free_mem_load(edev);
2543 err1:
2544         edev->ops->common->set_fp_int(edev->cdev, 0);
2545         qede_free_fp_array(edev);
2546         edev->num_queues = 0;
2547         edev->fp_num_tx = 0;
2548         edev->fp_num_rx = 0;
2549 out:
2550         if (!is_locked)
2551                 __qede_unlock(edev);
2552
2553         return rc;
2554 }
2555
2556 /* 'func' should be able to run between unload and reload assuming interface
2557  * is actually running, or afterwards in case it's currently DOWN.
2558  */
2559 void qede_reload(struct qede_dev *edev,
2560                  struct qede_reload_args *args, bool is_locked)
2561 {
2562         if (!is_locked)
2563                 __qede_lock(edev);
2564
2565         /* Since qede_lock is held, internal state wouldn't change even
2566          * if netdev state would start transitioning. Check whether current
2567          * internal configuration indicates device is up, then reload.
2568          */
2569         if (edev->state == QEDE_STATE_OPEN) {
2570                 qede_unload(edev, QEDE_UNLOAD_NORMAL, true);
2571                 if (args)
2572                         args->func(edev, args);
2573                 qede_load(edev, QEDE_LOAD_RELOAD, true);
2574
2575                 /* Since no one is going to do it for us, re-configure */
2576                 qede_config_rx_mode(edev->ndev);
2577         } else if (args) {
2578                 args->func(edev, args);
2579         }
2580
2581         if (!is_locked)
2582                 __qede_unlock(edev);
2583 }
2584
2585 /* called with rtnl_lock */
2586 static int qede_open(struct net_device *ndev)
2587 {
2588         struct qede_dev *edev = netdev_priv(ndev);
2589         int rc;
2590
2591         netif_carrier_off(ndev);
2592
2593         edev->ops->common->set_power_state(edev->cdev, PCI_D0);
2594
2595         rc = qede_load(edev, QEDE_LOAD_NORMAL, false);
2596         if (rc)
2597                 return rc;
2598
2599         udp_tunnel_nic_reset_ntf(ndev);
2600
2601         edev->ops->common->update_drv_state(edev->cdev, true);
2602
2603         return 0;
2604 }
2605
2606 static int qede_close(struct net_device *ndev)
2607 {
2608         struct qede_dev *edev = netdev_priv(ndev);
2609
2610         qede_unload(edev, QEDE_UNLOAD_NORMAL, false);
2611
2612         if (edev->cdev)
2613                 edev->ops->common->update_drv_state(edev->cdev, false);
2614
2615         return 0;
2616 }
2617
2618 static void qede_link_update(void *dev, struct qed_link_output *link)
2619 {
2620         struct qede_dev *edev = dev;
2621
2622         if (!test_bit(QEDE_FLAGS_LINK_REQUESTED, &edev->flags)) {
2623                 DP_VERBOSE(edev, NETIF_MSG_LINK, "Interface is not ready\n");
2624                 return;
2625         }
2626
2627         if (link->link_up) {
2628                 if (!netif_carrier_ok(edev->ndev)) {
2629                         DP_NOTICE(edev, "Link is up\n");
2630                         netif_tx_start_all_queues(edev->ndev);
2631                         netif_carrier_on(edev->ndev);
2632                         qede_rdma_dev_event_open(edev);
2633                 }
2634         } else {
2635                 if (netif_carrier_ok(edev->ndev)) {
2636                         DP_NOTICE(edev, "Link is down\n");
2637                         netif_tx_disable(edev->ndev);
2638                         netif_carrier_off(edev->ndev);
2639                         qede_rdma_dev_event_close(edev);
2640                 }
2641         }
2642 }
2643
2644 static void qede_schedule_recovery_handler(void *dev)
2645 {
2646         struct qede_dev *edev = dev;
2647
2648         if (edev->state == QEDE_STATE_RECOVERY) {
2649                 DP_NOTICE(edev,
2650                           "Avoid scheduling a recovery handling since already in recovery state\n");
2651                 return;
2652         }
2653
2654         set_bit(QEDE_SP_RECOVERY, &edev->sp_flags);
2655         schedule_delayed_work(&edev->sp_task, 0);
2656
2657         DP_INFO(edev, "Scheduled a recovery handler\n");
2658 }
2659
2660 static void qede_recovery_failed(struct qede_dev *edev)
2661 {
2662         netdev_err(edev->ndev, "Recovery handling has failed. Power cycle is needed.\n");
2663
2664         netif_device_detach(edev->ndev);
2665
2666         if (edev->cdev)
2667                 edev->ops->common->set_power_state(edev->cdev, PCI_D3hot);
2668 }
2669
2670 static void qede_recovery_handler(struct qede_dev *edev)
2671 {
2672         u32 curr_state = edev->state;
2673         int rc;
2674
2675         DP_NOTICE(edev, "Starting a recovery process\n");
2676
2677         /* No need to acquire first the qede_lock since is done by qede_sp_task
2678          * before calling this function.
2679          */
2680         edev->state = QEDE_STATE_RECOVERY;
2681
2682         edev->ops->common->recovery_prolog(edev->cdev);
2683
2684         if (curr_state == QEDE_STATE_OPEN)
2685                 qede_unload(edev, QEDE_UNLOAD_RECOVERY, true);
2686
2687         __qede_remove(edev->pdev, QEDE_REMOVE_RECOVERY);
2688
2689         rc = __qede_probe(edev->pdev, edev->dp_module, edev->dp_level,
2690                           IS_VF(edev), QEDE_PROBE_RECOVERY);
2691         if (rc) {
2692                 edev->cdev = NULL;
2693                 goto err;
2694         }
2695
2696         if (curr_state == QEDE_STATE_OPEN) {
2697                 rc = qede_load(edev, QEDE_LOAD_RECOVERY, true);
2698                 if (rc)
2699                         goto err;
2700
2701                 qede_config_rx_mode(edev->ndev);
2702                 udp_tunnel_nic_reset_ntf(edev->ndev);
2703         }
2704
2705         edev->state = curr_state;
2706
2707         DP_NOTICE(edev, "Recovery handling is done\n");
2708
2709         return;
2710
2711 err:
2712         qede_recovery_failed(edev);
2713 }
2714
2715 static void qede_atomic_hw_err_handler(struct qede_dev *edev)
2716 {
2717         struct qed_dev *cdev = edev->cdev;
2718
2719         DP_NOTICE(edev,
2720                   "Generic non-sleepable HW error handling started - err_flags 0x%lx\n",
2721                   edev->err_flags);
2722
2723         /* Get a call trace of the flow that led to the error */
2724         WARN_ON(test_bit(QEDE_ERR_WARN, &edev->err_flags));
2725
2726         /* Prevent HW attentions from being reasserted */
2727         if (test_bit(QEDE_ERR_ATTN_CLR_EN, &edev->err_flags))
2728                 edev->ops->common->attn_clr_enable(cdev, true);
2729
2730         DP_NOTICE(edev, "Generic non-sleepable HW error handling is done\n");
2731 }
2732
2733 static void qede_generic_hw_err_handler(struct qede_dev *edev)
2734 {
2735         DP_NOTICE(edev,
2736                   "Generic sleepable HW error handling started - err_flags 0x%lx\n",
2737                   edev->err_flags);
2738
2739         if (edev->devlink) {
2740                 DP_NOTICE(edev, "Reporting fatal error to devlink\n");
2741                 edev->ops->common->report_fatal_error(edev->devlink, edev->last_err_type);
2742         }
2743
2744         clear_bit(QEDE_ERR_IS_HANDLED, &edev->err_flags);
2745
2746         DP_NOTICE(edev, "Generic sleepable HW error handling is done\n");
2747 }
2748
2749 static void qede_set_hw_err_flags(struct qede_dev *edev,
2750                                   enum qed_hw_err_type err_type)
2751 {
2752         unsigned long err_flags = 0;
2753
2754         switch (err_type) {
2755         case QED_HW_ERR_DMAE_FAIL:
2756                 set_bit(QEDE_ERR_WARN, &err_flags);
2757                 fallthrough;
2758         case QED_HW_ERR_MFW_RESP_FAIL:
2759         case QED_HW_ERR_HW_ATTN:
2760         case QED_HW_ERR_RAMROD_FAIL:
2761         case QED_HW_ERR_FW_ASSERT:
2762                 set_bit(QEDE_ERR_ATTN_CLR_EN, &err_flags);
2763                 set_bit(QEDE_ERR_GET_DBG_INFO, &err_flags);
2764                 /* make this error as recoverable and start recovery*/
2765                 set_bit(QEDE_ERR_IS_RECOVERABLE, &err_flags);
2766                 break;
2767
2768         default:
2769                 DP_NOTICE(edev, "Unexpected HW error [%d]\n", err_type);
2770                 break;
2771         }
2772
2773         edev->err_flags |= err_flags;
2774 }
2775
2776 static void qede_schedule_hw_err_handler(void *dev,
2777                                          enum qed_hw_err_type err_type)
2778 {
2779         struct qede_dev *edev = dev;
2780
2781         /* Fan failure cannot be masked by handling of another HW error or by a
2782          * concurrent recovery process.
2783          */
2784         if ((test_and_set_bit(QEDE_ERR_IS_HANDLED, &edev->err_flags) ||
2785              edev->state == QEDE_STATE_RECOVERY) &&
2786              err_type != QED_HW_ERR_FAN_FAIL) {
2787                 DP_INFO(edev,
2788                         "Avoid scheduling an error handling while another HW error is being handled\n");
2789                 return;
2790         }
2791
2792         if (err_type >= QED_HW_ERR_LAST) {
2793                 DP_NOTICE(edev, "Unknown HW error [%d]\n", err_type);
2794                 clear_bit(QEDE_ERR_IS_HANDLED, &edev->err_flags);
2795                 return;
2796         }
2797
2798         edev->last_err_type = err_type;
2799         qede_set_hw_err_flags(edev, err_type);
2800         qede_atomic_hw_err_handler(edev);
2801         set_bit(QEDE_SP_HW_ERR, &edev->sp_flags);
2802         schedule_delayed_work(&edev->sp_task, 0);
2803
2804         DP_INFO(edev, "Scheduled a error handler [err_type %d]\n", err_type);
2805 }
2806
2807 static bool qede_is_txq_full(struct qede_dev *edev, struct qede_tx_queue *txq)
2808 {
2809         struct netdev_queue *netdev_txq;
2810
2811         netdev_txq = netdev_get_tx_queue(edev->ndev, txq->ndev_txq_id);
2812         if (netif_xmit_stopped(netdev_txq))
2813                 return true;
2814
2815         return false;
2816 }
2817
2818 static void qede_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data)
2819 {
2820         struct qede_dev *edev = dev;
2821         struct netdev_hw_addr *ha;
2822         int i;
2823
2824         if (edev->ndev->features & NETIF_F_IP_CSUM)
2825                 data->feat_flags |= QED_TLV_IP_CSUM;
2826         if (edev->ndev->features & NETIF_F_TSO)
2827                 data->feat_flags |= QED_TLV_LSO;
2828
2829         ether_addr_copy(data->mac[0], edev->ndev->dev_addr);
2830         eth_zero_addr(data->mac[1]);
2831         eth_zero_addr(data->mac[2]);
2832         /* Copy the first two UC macs */
2833         netif_addr_lock_bh(edev->ndev);
2834         i = 1;
2835         netdev_for_each_uc_addr(ha, edev->ndev) {
2836                 ether_addr_copy(data->mac[i++], ha->addr);
2837                 if (i == QED_TLV_MAC_COUNT)
2838                         break;
2839         }
2840
2841         netif_addr_unlock_bh(edev->ndev);
2842 }
2843
2844 static void qede_get_eth_tlv_data(void *dev, void *data)
2845 {
2846         struct qed_mfw_tlv_eth *etlv = data;
2847         struct qede_dev *edev = dev;
2848         struct qede_fastpath *fp;
2849         int i;
2850
2851         etlv->lso_maxoff_size = 0XFFFF;
2852         etlv->lso_maxoff_size_set = true;
2853         etlv->lso_minseg_size = (u16)ETH_TX_LSO_WINDOW_MIN_LEN;
2854         etlv->lso_minseg_size_set = true;
2855         etlv->prom_mode = !!(edev->ndev->flags & IFF_PROMISC);
2856         etlv->prom_mode_set = true;
2857         etlv->tx_descr_size = QEDE_TSS_COUNT(edev);
2858         etlv->tx_descr_size_set = true;
2859         etlv->rx_descr_size = QEDE_RSS_COUNT(edev);
2860         etlv->rx_descr_size_set = true;
2861         etlv->iov_offload = QED_MFW_TLV_IOV_OFFLOAD_VEB;
2862         etlv->iov_offload_set = true;
2863
2864         /* Fill information regarding queues; Should be done under the qede
2865          * lock to guarantee those don't change beneath our feet.
2866          */
2867         etlv->txqs_empty = true;
2868         etlv->rxqs_empty = true;
2869         etlv->num_txqs_full = 0;
2870         etlv->num_rxqs_full = 0;
2871
2872         __qede_lock(edev);
2873         for_each_queue(i) {
2874                 fp = &edev->fp_array[i];
2875                 if (fp->type & QEDE_FASTPATH_TX) {
2876                         struct qede_tx_queue *txq = QEDE_FP_TC0_TXQ(fp);
2877
2878                         if (txq->sw_tx_cons != txq->sw_tx_prod)
2879                                 etlv->txqs_empty = false;
2880                         if (qede_is_txq_full(edev, txq))
2881                                 etlv->num_txqs_full++;
2882                 }
2883                 if (fp->type & QEDE_FASTPATH_RX) {
2884                         if (qede_has_rx_work(fp->rxq))
2885                                 etlv->rxqs_empty = false;
2886
2887                         /* This one is a bit tricky; Firmware might stop
2888                          * placing packets if ring is not yet full.
2889                          * Give an approximation.
2890                          */
2891                         if (le16_to_cpu(*fp->rxq->hw_cons_ptr) -
2892                             qed_chain_get_cons_idx(&fp->rxq->rx_comp_ring) >
2893                             RX_RING_SIZE - 100)
2894                                 etlv->num_rxqs_full++;
2895                 }
2896         }
2897         __qede_unlock(edev);
2898
2899         etlv->txqs_empty_set = true;
2900         etlv->rxqs_empty_set = true;
2901         etlv->num_txqs_full_set = true;
2902         etlv->num_rxqs_full_set = true;
2903 }
2904
2905 /**
2906  * qede_io_error_detected(): Called when PCI error is detected
2907  *
2908  * @pdev: Pointer to PCI device
2909  * @state: The current pci connection state
2910  *
2911  *Return: pci_ers_result_t.
2912  *
2913  * This function is called after a PCI bus error affecting
2914  * this device has been detected.
2915  */
2916 static pci_ers_result_t
2917 qede_io_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
2918 {
2919         struct net_device *dev = pci_get_drvdata(pdev);
2920         struct qede_dev *edev = netdev_priv(dev);
2921
2922         if (!edev)
2923                 return PCI_ERS_RESULT_NONE;
2924
2925         DP_NOTICE(edev, "IO error detected [%d]\n", state);
2926
2927         __qede_lock(edev);
2928         if (edev->state == QEDE_STATE_RECOVERY) {
2929                 DP_NOTICE(edev, "Device already in the recovery state\n");
2930                 __qede_unlock(edev);
2931                 return PCI_ERS_RESULT_NONE;
2932         }
2933
2934         /* PF handles the recovery of its VFs */
2935         if (IS_VF(edev)) {
2936                 DP_VERBOSE(edev, QED_MSG_IOV,
2937                            "VF recovery is handled by its PF\n");
2938                 __qede_unlock(edev);
2939                 return PCI_ERS_RESULT_RECOVERED;
2940         }
2941
2942         /* Close OS Tx */
2943         netif_tx_disable(edev->ndev);
2944         netif_carrier_off(edev->ndev);
2945
2946         set_bit(QEDE_SP_AER, &edev->sp_flags);
2947         schedule_delayed_work(&edev->sp_task, 0);
2948
2949         __qede_unlock(edev);
2950
2951         return PCI_ERS_RESULT_CAN_RECOVER;
2952 }