drivers/net/ethernet/hisilicon/hns3/hns3_enet.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 // Copyright (c) 2016-2017 Hisilicon Limited.
   3
   4 #include <linux/dma-mapping.h>
   5 #include <linux/etherdevice.h>
   6 #include <linux/interrupt.h>
   7 #include <linux/if_vlan.h>
   8 #include <linux/ip.h>
   9 #include <linux/ipv6.h>
  10 #include <linux/module.h>
  11 #include <linux/pci.h>
  12 #include <linux/aer.h>
  13 #include <linux/skbuff.h>
  14 #include <linux/sctp.h>
  15 #include <linux/vermagic.h>
  16 #include <net/gre.h>
  17 #include <net/pkt_cls.h>
  18 #include <net/vxlan.h>
  19
  20 #include "hnae3.h"
  21 #include "hns3_enet.h"
  22
  23 static void hns3_clear_all_ring(struct hnae3_handle *h);
  24 static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h);
  25 static void hns3_remove_hw_addr(struct net_device *netdev);
  26
  27 static const char hns3_driver_name[] = "hns3";
  28 const char hns3_driver_version[] = VERMAGIC_STRING;
  29 static const char hns3_driver_string[] =
  30                         "Hisilicon Ethernet Network Driver for Hip08 Family";
  31 static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
  32 static struct hnae3_client client;
  33
  34 /* hns3_pci_tbl - PCI Device ID Table
  35  *
  36  * Last entry must be all 0s
  37  *
  38  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  39  *   Class, Class Mask, private data (not used) }
  40  */
  41 static const struct pci_device_id hns3_pci_tbl[] = {
  42         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
  43         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
  44         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
  45          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  46         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
  47          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  48         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
  49          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  50         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
  51          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  52         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
  53          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  54         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
  55         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF),
  56          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  57         /* required last entry */
  58         {0, }
  59 };
  60 MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
  61
  62 static irqreturn_t hns3_irq_handle(int irq, void *vector)
  63 {
  64         struct hns3_enet_tqp_vector *tqp_vector = vector;
  65
  66         napi_schedule(&tqp_vector->napi);
  67
  68         return IRQ_HANDLED;
  69 }
  70
  71 /* This callback function is used to set affinity changes to the irq affinity
  72  * masks when the irq_set_affinity_notifier function is used.
  73  */
  74 static void hns3_nic_irq_affinity_notify(struct irq_affinity_notify *notify,
  75                                          const cpumask_t *mask)
  76 {
  77         struct hns3_enet_tqp_vector *tqp_vectors =
  78                 container_of(notify, struct hns3_enet_tqp_vector,
  79                              affinity_notify);
  80
  81         tqp_vectors->affinity_mask = *mask;
  82 }
  83
  84 static void hns3_nic_irq_affinity_release(struct kref *ref)
  85 {
  86 }
  87
  88 static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
  89 {
  90         struct hns3_enet_tqp_vector *tqp_vectors;
  91         unsigned int i;
  92
  93         for (i = 0; i < priv->vector_num; i++) {
  94                 tqp_vectors = &priv->tqp_vector[i];
  95
  96                 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
  97                         continue;
  98
  99                 /* clear the affinity notifier and affinity mask */
 100                 irq_set_affinity_notifier(tqp_vectors->vector_irq, NULL);
 101                 irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);
 102
 103                 /* release the irq resource */
 104                 free_irq(tqp_vectors->vector_irq, tqp_vectors);
 105                 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
 106         }
 107 }
 108
 109 static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
 110 {
 111         struct hns3_enet_tqp_vector *tqp_vectors;
 112         int txrx_int_idx = 0;
 113         int rx_int_idx = 0;
 114         int tx_int_idx = 0;
 115         unsigned int i;
 116         int ret;
 117
 118         for (i = 0; i < priv->vector_num; i++) {
 119                 tqp_vectors = &priv->tqp_vector[i];
 120
 121                 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
 122                         continue;
 123
 124                 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
 125                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
 126                                  "%s-%s-%d", priv->netdev->name, "TxRx",
 127                                  txrx_int_idx++);
 128                         txrx_int_idx++;
 129                 } else if (tqp_vectors->rx_group.ring) {
 130                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
 131                                  "%s-%s-%d", priv->netdev->name, "Rx",
 132                                  rx_int_idx++);
 133                 } else if (tqp_vectors->tx_group.ring) {
 134                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
 135                                  "%s-%s-%d", priv->netdev->name, "Tx",
 136                                  tx_int_idx++);
 137                 } else {
 138                         /* Skip this unused q_vector */
 139                         continue;
 140                 }
 141
 142                 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
 143
 144                 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
 145                                   tqp_vectors->name,
 146                                        tqp_vectors);
 147                 if (ret) {
 148                         netdev_err(priv->netdev, "request irq(%d) fail\n",
 149                                    tqp_vectors->vector_irq);
 150                         return ret;
 151                 }
 152
 153                 tqp_vectors->affinity_notify.notify =
 154                                         hns3_nic_irq_affinity_notify;
 155                 tqp_vectors->affinity_notify.release =
 156                                         hns3_nic_irq_affinity_release;
 157                 irq_set_affinity_notifier(tqp_vectors->vector_irq,
 158                                           &tqp_vectors->affinity_notify);
 159                 irq_set_affinity_hint(tqp_vectors->vector_irq,
 160                                       &tqp_vectors->affinity_mask);
 161
 162                 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
 163         }
 164
 165         return 0;
 166 }
 167
 168 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
 169                                  u32 mask_en)
 170 {
 171         writel(mask_en, tqp_vector->mask_addr);
 172 }
 173
 174 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
 175 {
 176         napi_enable(&tqp_vector->napi);
 177
 178         /* enable vector */
 179         hns3_mask_vector_irq(tqp_vector, 1);
 180 }
 181
 182 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
 183 {
 184         /* disable vector */
 185         hns3_mask_vector_irq(tqp_vector, 0);
 186
 187         disable_irq(tqp_vector->vector_irq);
 188         napi_disable(&tqp_vector->napi);
 189 }
 190
 191 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
 192                                  u32 rl_value)
 193 {
 194         u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
 195
 196         /* this defines the configuration for RL (Interrupt Rate Limiter).
 197          * Rl defines rate of interrupts i.e. number of interrupts-per-second
 198          * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
 199          */
 200
 201         if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&
 202             !tqp_vector->rx_group.coal.gl_adapt_enable)
 203                 /* According to the hardware, the range of rl_reg is
 204                  * 0-59 and the unit is 4.
 205                  */
 206                 rl_reg |=  HNS3_INT_RL_ENABLE_MASK;
 207
 208         writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
 209 }
 210
 211 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 212                                     u32 gl_value)
 213 {
 214         u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
 215
 216         writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
 217 }
 218
 219 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 220                                     u32 gl_value)
 221 {
 222         u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
 223
 224         writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
 225 }
 226
 227 static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
 228                                    struct hns3_nic_priv *priv)
 229 {
 230         /* initialize the configuration for interrupt coalescing.
 231          * 1. GL (Interrupt Gap Limiter)
 232          * 2. RL (Interrupt Rate Limiter)
 233          */
 234
 235         /* Default: enable interrupt coalescing self-adaptive and GL */
 236         tqp_vector->tx_group.coal.gl_adapt_enable = 1;
 237         tqp_vector->rx_group.coal.gl_adapt_enable = 1;
 238
 239         tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;
 240         tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;
 241
 242         tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
 243         tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;
 244         tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;
 245 }
 246
 247 static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
 248                                       struct hns3_nic_priv *priv)
 249 {
 250         struct hnae3_handle *h = priv->ae_handle;
 251
 252         hns3_set_vector_coalesce_tx_gl(tqp_vector,
 253                                        tqp_vector->tx_group.coal.int_gl);
 254         hns3_set_vector_coalesce_rx_gl(tqp_vector,
 255                                        tqp_vector->rx_group.coal.int_gl);
 256         hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
 257 }
 258
 259 static int hns3_nic_set_real_num_queue(struct net_device *netdev)
 260 {
 261         struct hnae3_handle *h = hns3_get_handle(netdev);
 262         struct hnae3_knic_private_info *kinfo = &h->kinfo;
 263         unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
 264         int i, ret;
 265
 266         if (kinfo->num_tc <= 1) {
 267                 netdev_reset_tc(netdev);
 268         } else {
 269                 ret = netdev_set_num_tc(netdev, kinfo->num_tc);
 270                 if (ret) {
 271                         netdev_err(netdev,
 272                                    "netdev_set_num_tc fail, ret=%d!\n", ret);
 273                         return ret;
 274                 }
 275
 276                 for (i = 0; i < HNAE3_MAX_TC; i++) {
 277                         if (!kinfo->tc_info[i].enable)
 278                                 continue;
 279
 280                         netdev_set_tc_queue(netdev,
 281                                             kinfo->tc_info[i].tc,
 282                                             kinfo->tc_info[i].tqp_count,
 283                                             kinfo->tc_info[i].tqp_offset);
 284                 }
 285         }
 286
 287         ret = netif_set_real_num_tx_queues(netdev, queue_size);
 288         if (ret) {
 289                 netdev_err(netdev,
 290                            "netif_set_real_num_tx_queues fail, ret=%d!\n",
 291                            ret);
 292                 return ret;
 293         }
 294
 295         ret = netif_set_real_num_rx_queues(netdev, queue_size);
 296         if (ret) {
 297                 netdev_err(netdev,
 298                            "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
 299                 return ret;
 300         }
 301
 302         return 0;
 303 }
 304
 305 static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
 306 {
 307         u16 alloc_tqps, max_rss_size, rss_size;
 308
 309         h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);
 310         rss_size = alloc_tqps / h->kinfo.num_tc;
 311
 312         return min_t(u16, rss_size, max_rss_size);
 313 }
 314
 315 static int hns3_nic_net_up(struct net_device *netdev)
 316 {
 317         struct hns3_nic_priv *priv = netdev_priv(netdev);
 318         struct hnae3_handle *h = priv->ae_handle;
 319         int i, j;
 320         int ret;
 321
 322         ret = hns3_nic_reset_all_ring(h);
 323         if (ret)
 324                 return ret;
 325
 326         /* get irq resource for all vectors */
 327         ret = hns3_nic_init_irq(priv);
 328         if (ret) {
 329                 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
 330                 return ret;
 331         }
 332
 333         /* enable the vectors */
 334         for (i = 0; i < priv->vector_num; i++)
 335                 hns3_vector_enable(&priv->tqp_vector[i]);
 336
 337         /* start the ae_dev */
 338         ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
 339         if (ret)
 340                 goto out_start_err;
 341
 342         clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
 343
 344         return 0;
 345
 346 out_start_err:
 347         for (j = i - 1; j >= 0; j--)
 348                 hns3_vector_disable(&priv->tqp_vector[j]);
 349
 350         hns3_nic_uninit_irq(priv);
 351
 352         return ret;
 353 }
 354
 355 static int hns3_nic_net_open(struct net_device *netdev)
 356 {
 357         struct hns3_nic_priv *priv = netdev_priv(netdev);
 358         struct hnae3_handle *h = hns3_get_handle(netdev);
 359         struct hnae3_knic_private_info *kinfo;
 360         int i, ret;
 361
 362         netif_carrier_off(netdev);
 363
 364         ret = hns3_nic_set_real_num_queue(netdev);
 365         if (ret)
 366                 return ret;
 367
 368         ret = hns3_nic_net_up(netdev);
 369         if (ret) {
 370                 netdev_err(netdev,
 371                            "hns net up fail, ret=%d!\n", ret);
 372                 return ret;
 373         }
 374
 375         kinfo = &h->kinfo;
 376         for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
 377                 netdev_set_prio_tc_map(netdev, i,
 378                                        kinfo->prio_tc[i]);
 379         }
 380
 381         priv->ae_handle->last_reset_time = jiffies;
 382         return 0;
 383 }
 384
 385 static void hns3_nic_net_down(struct net_device *netdev)
 386 {
 387         struct hns3_nic_priv *priv = netdev_priv(netdev);
 388         const struct hnae3_ae_ops *ops;
 389         int i;
 390
 391         if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
 392                 return;
 393
 394         /* disable vectors */
 395         for (i = 0; i < priv->vector_num; i++)
 396                 hns3_vector_disable(&priv->tqp_vector[i]);
 397
 398         /* stop ae_dev */
 399         ops = priv->ae_handle->ae_algo->ops;
 400         if (ops->stop)
 401                 ops->stop(priv->ae_handle);
 402
 403         /* free irq resources */
 404         hns3_nic_uninit_irq(priv);
 405
 406         hns3_clear_all_ring(priv->ae_handle);
 407 }
 408
 409 static int hns3_nic_net_stop(struct net_device *netdev)
 410 {
 411         netif_tx_stop_all_queues(netdev);
 412         netif_carrier_off(netdev);
 413
 414         hns3_nic_net_down(netdev);
 415
 416         return 0;
 417 }
 418
 419 static int hns3_nic_uc_sync(struct net_device *netdev,
 420                             const unsigned char *addr)
 421 {
 422         struct hnae3_handle *h = hns3_get_handle(netdev);
 423
 424         if (h->ae_algo->ops->add_uc_addr)
 425                 return h->ae_algo->ops->add_uc_addr(h, addr);
 426
 427         return 0;
 428 }
 429
 430 static int hns3_nic_uc_unsync(struct net_device *netdev,
 431                               const unsigned char *addr)
 432 {
 433         struct hnae3_handle *h = hns3_get_handle(netdev);
 434
 435         if (h->ae_algo->ops->rm_uc_addr)
 436                 return h->ae_algo->ops->rm_uc_addr(h, addr);
 437
 438         return 0;
 439 }
 440
 441 static int hns3_nic_mc_sync(struct net_device *netdev,
 442                             const unsigned char *addr)
 443 {
 444         struct hnae3_handle *h = hns3_get_handle(netdev);
 445
 446         if (h->ae_algo->ops->add_mc_addr)
 447                 return h->ae_algo->ops->add_mc_addr(h, addr);
 448
 449         return 0;
 450 }
 451
 452 static int hns3_nic_mc_unsync(struct net_device *netdev,
 453                               const unsigned char *addr)
 454 {
 455         struct hnae3_handle *h = hns3_get_handle(netdev);
 456
 457         if (h->ae_algo->ops->rm_mc_addr)
 458                 return h->ae_algo->ops->rm_mc_addr(h, addr);
 459
 460         return 0;
 461 }
 462
 463 static u8 hns3_get_netdev_flags(struct net_device *netdev)
 464 {
 465         u8 flags = 0;
 466
 467         if (netdev->flags & IFF_PROMISC) {
 468                 flags = HNAE3_USER_UPE | HNAE3_USER_MPE;
 469         } else {
 470                 flags |= HNAE3_VLAN_FLTR;
 471                 if (netdev->flags & IFF_ALLMULTI)
 472                         flags |= HNAE3_USER_MPE;
 473         }
 474
 475         return flags;
 476 }
 477
 478 static void hns3_nic_set_rx_mode(struct net_device *netdev)
 479 {
 480         struct hnae3_handle *h = hns3_get_handle(netdev);
 481         u8 new_flags;
 482         int ret;
 483
 484         new_flags = hns3_get_netdev_flags(netdev);
 485
 486         ret = __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);
 487         if (ret) {
 488                 netdev_err(netdev, "sync uc address fail\n");
 489                 if (ret == -ENOSPC)
 490                         new_flags |= HNAE3_OVERFLOW_UPE;
 491         }
 492
 493         if (netdev->flags & IFF_MULTICAST) {
 494                 ret = __dev_mc_sync(netdev, hns3_nic_mc_sync,
 495                                     hns3_nic_mc_unsync);
 496                 if (ret) {
 497                         netdev_err(netdev, "sync mc address fail\n");
 498                         if (ret == -ENOSPC)
 499                                 new_flags |= HNAE3_OVERFLOW_MPE;
 500                 }
 501         }
 502
 503         hns3_update_promisc_mode(netdev, new_flags);
 504         /* User mode Promisc mode enable and vlan filtering is disabled to
 505          * let all packets in. MAC-VLAN Table overflow Promisc enabled and
 506          * vlan fitering is enabled
 507          */
 508         hns3_enable_vlan_filter(netdev, new_flags & HNAE3_VLAN_FLTR);
 509         h->netdev_flags = new_flags;
 510 }
 511
 512 void hns3_update_promisc_mode(struct net_device *netdev, u8 promisc_flags)
 513 {
 514         struct hns3_nic_priv *priv = netdev_priv(netdev);
 515         struct hnae3_handle *h = priv->ae_handle;
 516
 517         if (h->ae_algo->ops->set_promisc_mode) {
 518                 h->ae_algo->ops->set_promisc_mode(h,
 519                                                   promisc_flags & HNAE3_UPE,
 520                                                   promisc_flags & HNAE3_MPE);
 521         }
 522 }
 523
 524 void hns3_enable_vlan_filter(struct net_device *netdev, bool enable)
 525 {
 526         struct hns3_nic_priv *priv = netdev_priv(netdev);
 527         struct hnae3_handle *h = priv->ae_handle;
 528         bool last_state;
 529
 530         if (h->pdev->revision >= 0x21 && h->ae_algo->ops->enable_vlan_filter) {
 531                 last_state = h->netdev_flags & HNAE3_VLAN_FLTR ? true : false;
 532                 if (enable != last_state) {
 533                         netdev_info(netdev,
 534                                     "%s vlan filter\n",
 535                                     enable ? "enable" : "disable");
 536                         h->ae_algo->ops->enable_vlan_filter(h, enable);
 537                 }
 538         }
 539 }
 540
 541 static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
 542                         u16 *mss, u32 *type_cs_vlan_tso)
 543 {
 544         u32 l4_offset, hdr_len;
 545         union l3_hdr_info l3;
 546         union l4_hdr_info l4;
 547         u32 l4_paylen;
 548         int ret;
 549
 550         if (!skb_is_gso(skb))
 551                 return 0;
 552
 553         ret = skb_cow_head(skb, 0);
 554         if (ret)
 555                 return ret;
 556
 557         l3.hdr = skb_network_header(skb);
 558         l4.hdr = skb_transport_header(skb);
 559
 560         /* Software should clear the IPv4's checksum field when tso is
 561          * needed.
 562          */
 563         if (l3.v4->version == 4)
 564                 l3.v4->check = 0;
 565
 566         /* tunnel packet.*/
 567         if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
 568                                          SKB_GSO_GRE_CSUM |
 569                                          SKB_GSO_UDP_TUNNEL |
 570                                          SKB_GSO_UDP_TUNNEL_CSUM)) {
 571                 if ((!(skb_shinfo(skb)->gso_type &
 572                     SKB_GSO_PARTIAL)) &&
 573                     (skb_shinfo(skb)->gso_type &
 574                     SKB_GSO_UDP_TUNNEL_CSUM)) {
 575                         /* Software should clear the udp's checksum
 576                          * field when tso is needed.
 577                          */
 578                         l4.udp->check = 0;
 579                 }
 580                 /* reset l3&l4 pointers from outer to inner headers */
 581                 l3.hdr = skb_inner_network_header(skb);
 582                 l4.hdr = skb_inner_transport_header(skb);
 583
 584                 /* Software should clear the IPv4's checksum field when
 585                  * tso is needed.
 586                  */
 587                 if (l3.v4->version == 4)
 588                         l3.v4->check = 0;
 589         }
 590
 591         /* normal or tunnel packet*/
 592         l4_offset = l4.hdr - skb->data;
 593         hdr_len = (l4.tcp->doff * 4) + l4_offset;
 594
 595         /* remove payload length from inner pseudo checksum when tso*/
 596         l4_paylen = skb->len - l4_offset;
 597         csum_replace_by_diff(&l4.tcp->check,
 598                              (__force __wsum)htonl(l4_paylen));
 599
 600         /* find the txbd field values */
 601         *paylen = skb->len - hdr_len;
 602         hnae3_set_bit(*type_cs_vlan_tso,
 603                       HNS3_TXD_TSO_B, 1);
 604
 605         /* get MSS for TSO */
 606         *mss = skb_shinfo(skb)->gso_size;
 607
 608         return 0;
 609 }
 610
 611 static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
 612                                 u8 *il4_proto)
 613 {
 614         union {
 615                 struct iphdr *v4;
 616                 struct ipv6hdr *v6;
 617                 unsigned char *hdr;
 618         } l3;
 619         unsigned char *l4_hdr;
 620         unsigned char *exthdr;
 621         u8 l4_proto_tmp;
 622         __be16 frag_off;
 623
 624         /* find outer header point */
 625         l3.hdr = skb_network_header(skb);
 626         l4_hdr = skb_transport_header(skb);
 627
 628         if (skb->protocol == htons(ETH_P_IPV6)) {
 629                 exthdr = l3.hdr + sizeof(*l3.v6);
 630                 l4_proto_tmp = l3.v6->nexthdr;
 631                 if (l4_hdr != exthdr)
 632                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 633                                          &l4_proto_tmp, &frag_off);
 634         } else if (skb->protocol == htons(ETH_P_IP)) {
 635                 l4_proto_tmp = l3.v4->protocol;
 636         } else {
 637                 return -EINVAL;
 638         }
 639
 640         *ol4_proto = l4_proto_tmp;
 641
 642         /* tunnel packet */
 643         if (!skb->encapsulation) {
 644                 *il4_proto = 0;
 645                 return 0;
 646         }
 647
 648         /* find inner header point */
 649         l3.hdr = skb_inner_network_header(skb);
 650         l4_hdr = skb_inner_transport_header(skb);
 651
 652         if (l3.v6->version == 6) {
 653                 exthdr = l3.hdr + sizeof(*l3.v6);
 654                 l4_proto_tmp = l3.v6->nexthdr;
 655                 if (l4_hdr != exthdr)
 656                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 657                                          &l4_proto_tmp, &frag_off);
 658         } else if (l3.v4->version == 4) {
 659                 l4_proto_tmp = l3.v4->protocol;
 660         }
 661
 662         *il4_proto = l4_proto_tmp;
 663
 664         return 0;
 665 }
 666
 667 static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
 668                                 u8 il4_proto, u32 *type_cs_vlan_tso,
 669                                 u32 *ol_type_vlan_len_msec)
 670 {
 671         union {
 672                 struct iphdr *v4;
 673                 struct ipv6hdr *v6;
 674                 unsigned char *hdr;
 675         } l3;
 676         union {
 677                 struct tcphdr *tcp;
 678                 struct udphdr *udp;
 679                 struct gre_base_hdr *gre;
 680                 unsigned char *hdr;
 681         } l4;
 682         unsigned char *l2_hdr;
 683         u8 l4_proto = ol4_proto;
 684         u32 ol2_len;
 685         u32 ol3_len;
 686         u32 ol4_len;
 687         u32 l2_len;
 688         u32 l3_len;
 689
 690         l3.hdr = skb_network_header(skb);
 691         l4.hdr = skb_transport_header(skb);
 692
 693         /* compute L2 header size for normal packet, defined in 2 Bytes */
 694         l2_len = l3.hdr - skb->data;
 695         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
 696                         HNS3_TXD_L2LEN_S, l2_len >> 1);
 697
 698         /* tunnel packet*/
 699         if (skb->encapsulation) {
 700                 /* compute OL2 header size, defined in 2 Bytes */
 701                 ol2_len = l2_len;
 702                 hnae3_set_field(*ol_type_vlan_len_msec,
 703                                 HNS3_TXD_L2LEN_M,
 704                                 HNS3_TXD_L2LEN_S, ol2_len >> 1);
 705
 706                 /* compute OL3 header size, defined in 4 Bytes */
 707                 ol3_len = l4.hdr - l3.hdr;
 708                 hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
 709                                 HNS3_TXD_L3LEN_S, ol3_len >> 2);
 710
 711                 /* MAC in UDP, MAC in GRE (0x6558)*/
 712                 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
 713                         /* switch MAC header ptr from outer to inner header.*/
 714                         l2_hdr = skb_inner_mac_header(skb);
 715
 716                         /* compute OL4 header size, defined in 4 Bytes. */
 717                         ol4_len = l2_hdr - l4.hdr;
 718                         hnae3_set_field(*ol_type_vlan_len_msec,
 719                                         HNS3_TXD_L4LEN_M, HNS3_TXD_L4LEN_S,
 720                                         ol4_len >> 2);
 721
 722                         /* switch IP header ptr from outer to inner header */
 723                         l3.hdr = skb_inner_network_header(skb);
 724
 725                         /* compute inner l2 header size, defined in 2 Bytes. */
 726                         l2_len = l3.hdr - l2_hdr;
 727                         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
 728                                         HNS3_TXD_L2LEN_S, l2_len >> 1);
 729                 } else {
 730                         /* skb packet types not supported by hardware,
 731                          * txbd len fild doesn't be filled.
 732                          */
 733                         return;
 734                 }
 735
 736                 /* switch L4 header pointer from outer to inner */
 737                 l4.hdr = skb_inner_transport_header(skb);
 738
 739                 l4_proto = il4_proto;
 740         }
 741
 742         /* compute inner(/normal) L3 header size, defined in 4 Bytes */
 743         l3_len = l4.hdr - l3.hdr;
 744         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
 745                         HNS3_TXD_L3LEN_S, l3_len >> 2);
 746
 747         /* compute inner(/normal) L4 header size, defined in 4 Bytes */
 748         switch (l4_proto) {
 749         case IPPROTO_TCP:
 750                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 751                                 HNS3_TXD_L4LEN_S, l4.tcp->doff);
 752                 break;
 753         case IPPROTO_SCTP:
 754                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 755                                 HNS3_TXD_L4LEN_S,
 756                                 (sizeof(struct sctphdr) >> 2));
 757                 break;
 758         case IPPROTO_UDP:
 759                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 760                                 HNS3_TXD_L4LEN_S,
 761                                 (sizeof(struct udphdr) >> 2));
 762                 break;
 763         default:
 764                 /* skb packet types not supported by hardware,
 765                  * txbd len fild doesn't be filled.
 766                  */
 767                 return;
 768         }
 769 }
 770
 771 /* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
 772  * and it is udp packet, which has a dest port as the IANA assigned.
 773  * the hardware is expected to do the checksum offload, but the
 774  * hardware will not do the checksum offload when udp dest port is
 775  * 4789.
 776  */
 777 static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
 778 {
 779 #define IANA_VXLAN_PORT 4789
 780         union {
 781                 struct tcphdr *tcp;
 782                 struct udphdr *udp;
 783                 struct gre_base_hdr *gre;
 784                 unsigned char *hdr;
 785         } l4;
 786
 787         l4.hdr = skb_transport_header(skb);
 788
 789         if (!(!skb->encapsulation && l4.udp->dest == htons(IANA_VXLAN_PORT)))
 790                 return false;
 791
 792         skb_checksum_help(skb);
 793
 794         return true;
 795 }
 796
 797 static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
 798                                    u8 il4_proto, u32 *type_cs_vlan_tso,
 799                                    u32 *ol_type_vlan_len_msec)
 800 {
 801         union {
 802                 struct iphdr *v4;
 803                 struct ipv6hdr *v6;
 804                 unsigned char *hdr;
 805         } l3;
 806         u32 l4_proto = ol4_proto;
 807
 808         l3.hdr = skb_network_header(skb);
 809
 810         /* define OL3 type and tunnel type(OL4).*/
 811         if (skb->encapsulation) {
 812                 /* define outer network header type.*/
 813                 if (skb->protocol == htons(ETH_P_IP)) {
 814                         if (skb_is_gso(skb))
 815                                 hnae3_set_field(*ol_type_vlan_len_msec,
 816                                                 HNS3_TXD_OL3T_M,
 817                                                 HNS3_TXD_OL3T_S,
 818                                                 HNS3_OL3T_IPV4_CSUM);
 819                         else
 820                                 hnae3_set_field(*ol_type_vlan_len_msec,
 821                                                 HNS3_TXD_OL3T_M,
 822                                                 HNS3_TXD_OL3T_S,
 823                                                 HNS3_OL3T_IPV4_NO_CSUM);
 824
 825                 } else if (skb->protocol == htons(ETH_P_IPV6)) {
 826                         hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
 827                                         HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
 828                 }
 829
 830                 /* define tunnel type(OL4).*/
 831                 switch (l4_proto) {
 832                 case IPPROTO_UDP:
 833                         hnae3_set_field(*ol_type_vlan_len_msec,
 834                                         HNS3_TXD_TUNTYPE_M,
 835                                         HNS3_TXD_TUNTYPE_S,
 836                                         HNS3_TUN_MAC_IN_UDP);
 837                         break;
 838                 case IPPROTO_GRE:
 839                         hnae3_set_field(*ol_type_vlan_len_msec,
 840                                         HNS3_TXD_TUNTYPE_M,
 841                                         HNS3_TXD_TUNTYPE_S,
 842                                         HNS3_TUN_NVGRE);
 843                         break;
 844                 default:
 845                         /* drop the skb tunnel packet if hardware don't support,
 846                          * because hardware can't calculate csum when TSO.
 847                          */
 848                         if (skb_is_gso(skb))
 849                                 return -EDOM;
 850
 851                         /* the stack computes the IP header already,
 852                          * driver calculate l4 checksum when not TSO.
 853                          */
 854                         skb_checksum_help(skb);
 855                         return 0;
 856                 }
 857
 858                 l3.hdr = skb_inner_network_header(skb);
 859                 l4_proto = il4_proto;
 860         }
 861
 862         if (l3.v4->version == 4) {
 863                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
 864                                 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
 865
 866                 /* the stack computes the IP header already, the only time we
 867                  * need the hardware to recompute it is in the case of TSO.
 868                  */
 869                 if (skb_is_gso(skb))
 870                         hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
 871         } else if (l3.v6->version == 6) {
 872                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
 873                                 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
 874         }
 875
 876         switch (l4_proto) {
 877         case IPPROTO_TCP:
 878                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 879                 hnae3_set_field(*type_cs_vlan_tso,
 880                                 HNS3_TXD_L4T_M,
 881                                 HNS3_TXD_L4T_S,
 882                                 HNS3_L4T_TCP);
 883                 break;
 884         case IPPROTO_UDP:
 885                 if (hns3_tunnel_csum_bug(skb))
 886                         break;
 887
 888                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 889                 hnae3_set_field(*type_cs_vlan_tso,
 890                                 HNS3_TXD_L4T_M,
 891                                 HNS3_TXD_L4T_S,
 892                                 HNS3_L4T_UDP);
 893                 break;
 894         case IPPROTO_SCTP:
 895                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 896                 hnae3_set_field(*type_cs_vlan_tso,
 897                                 HNS3_TXD_L4T_M,
 898                                 HNS3_TXD_L4T_S,
 899                                 HNS3_L4T_SCTP);
 900                 break;
 901         default:
 902                 /* drop the skb tunnel packet if hardware don't support,
 903                  * because hardware can't calculate csum when TSO.
 904                  */
 905                 if (skb_is_gso(skb))
 906                         return -EDOM;
 907
 908                 /* the stack computes the IP header already,
 909                  * driver calculate l4 checksum when not TSO.
 910                  */
 911                 skb_checksum_help(skb);
 912                 return 0;
 913         }
 914
 915         return 0;
 916 }
 917
 918 static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
 919 {
 920         /* Config bd buffer end */
 921         hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
 922                         HNS3_TXD_BDTYPE_S, 0);
 923         hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
 924         hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
 925         hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
 926 }
 927
 928 static int hns3_fill_desc_vtags(struct sk_buff *skb,
 929                                 struct hns3_enet_ring *tx_ring,
 930                                 u32 *inner_vlan_flag,
 931                                 u32 *out_vlan_flag,
 932                                 u16 *inner_vtag,
 933                                 u16 *out_vtag)
 934 {
 935 #define HNS3_TX_VLAN_PRIO_SHIFT 13
 936
 937         if (skb->protocol == htons(ETH_P_8021Q) &&
 938             !(tx_ring->tqp->handle->kinfo.netdev->features &
 939             NETIF_F_HW_VLAN_CTAG_TX)) {
 940                 /* When HW VLAN acceleration is turned off, and the stack
 941                  * sets the protocol to 802.1q, the driver just need to
 942                  * set the protocol to the encapsulated ethertype.
 943                  */
 944                 skb->protocol = vlan_get_protocol(skb);
 945                 return 0;
 946         }
 947
 948         if (skb_vlan_tag_present(skb)) {
 949                 u16 vlan_tag;
 950
 951                 vlan_tag = skb_vlan_tag_get(skb);
 952                 vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
 953
 954                 /* Based on hw strategy, use out_vtag in two layer tag case,
 955                  * and use inner_vtag in one tag case.
 956                  */
 957                 if (skb->protocol == htons(ETH_P_8021Q)) {
 958                         hnae3_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
 959                         *out_vtag = vlan_tag;
 960                 } else {
 961                         hnae3_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
 962                         *inner_vtag = vlan_tag;
 963                 }
 964         } else if (skb->protocol == htons(ETH_P_8021Q)) {
 965                 struct vlan_ethhdr *vhdr;
 966                 int rc;
 967
 968                 rc = skb_cow_head(skb, 0);
 969                 if (rc < 0)
 970                         return rc;
 971                 vhdr = (struct vlan_ethhdr *)skb->data;
 972                 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
 973                                         << HNS3_TX_VLAN_PRIO_SHIFT);
 974         }
 975
 976         skb->protocol = vlan_get_protocol(skb);
 977         return 0;
 978 }
 979
 980 static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
 981                           int size, int frag_end, enum hns_desc_type type)
 982 {
 983         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
 984         struct hns3_desc *desc = &ring->desc[ring->next_to_use];
 985         struct device *dev = ring_to_dev(ring);
 986         u32 ol_type_vlan_len_msec = 0;
 987         u16 bdtp_fe_sc_vld_ra_ri = 0;
 988         struct skb_frag_struct *frag;
 989         unsigned int frag_buf_num;
 990         u32 type_cs_vlan_tso = 0;
 991         struct sk_buff *skb;
 992         u16 inner_vtag = 0;
 993         u16 out_vtag = 0;
 994         unsigned int k;
 995         int sizeoflast;
 996         u32 paylen = 0;
 997         dma_addr_t dma;
 998         u16 mss = 0;
 999         u8 ol4_proto;
1000         u8 il4_proto;
1001         int ret;
1002
1003         if (type == DESC_TYPE_SKB) {
1004                 skb = (struct sk_buff *)priv;
1005                 paylen = skb->len;
1006
1007                 ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
1008                                            &ol_type_vlan_len_msec,
1009                                            &inner_vtag, &out_vtag);
1010                 if (unlikely(ret))
1011                         return ret;
1012
1013                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1014                         skb_reset_mac_len(skb);
1015
1016                         ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
1017                         if (ret)
1018                                 return ret;
1019                         hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
1020                                             &type_cs_vlan_tso,
1021                                             &ol_type_vlan_len_msec);
1022                         ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
1023                                                       &type_cs_vlan_tso,
1024                                                       &ol_type_vlan_len_msec);
1025                         if (ret)
1026                                 return ret;
1027
1028                         ret = hns3_set_tso(skb, &paylen, &mss,
1029                                            &type_cs_vlan_tso);
1030                         if (ret)
1031                                 return ret;
1032                 }
1033
1034                 /* Set txbd */
1035                 desc->tx.ol_type_vlan_len_msec =
1036                         cpu_to_le32(ol_type_vlan_len_msec);
1037                 desc->tx.type_cs_vlan_tso_len =
1038                         cpu_to_le32(type_cs_vlan_tso);
1039                 desc->tx.paylen = cpu_to_le32(paylen);
1040                 desc->tx.mss = cpu_to_le16(mss);
1041                 desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
1042                 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
1043
1044                 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1045         } else {
1046                 frag = (struct skb_frag_struct *)priv;
1047                 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1048         }
1049
1050         if (dma_mapping_error(ring->dev, dma)) {
1051                 ring->stats.sw_err_cnt++;
1052                 return -ENOMEM;
1053         }
1054
1055         desc_cb->length = size;
1056
1057         frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1058         sizeoflast = size % HNS3_MAX_BD_SIZE;
1059         sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
1060
1061         /* When frag size is bigger than hardware limit, split this frag */
1062         for (k = 0; k < frag_buf_num; k++) {
1063                 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
1064                 desc_cb->priv = priv;
1065                 desc_cb->dma = dma + HNS3_MAX_BD_SIZE * k;
1066                 desc_cb->type = (type == DESC_TYPE_SKB && !k) ?
1067                                         DESC_TYPE_SKB : DESC_TYPE_PAGE;
1068
1069                 /* now, fill the descriptor */
1070                 desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);
1071                 desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?
1072                                 (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);
1073                 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri,
1074                                        frag_end && (k == frag_buf_num - 1) ?
1075                                                 1 : 0);
1076                 desc->tx.bdtp_fe_sc_vld_ra_ri =
1077                                 cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
1078
1079                 /* move ring pointer to next.*/
1080                 ring_ptr_move_fw(ring, next_to_use);
1081
1082                 desc_cb = &ring->desc_cb[ring->next_to_use];
1083                 desc = &ring->desc[ring->next_to_use];
1084         }
1085
1086         return 0;
1087 }
1088
1089 static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
1090                                    struct hns3_enet_ring *ring)
1091 {
1092         struct sk_buff *skb = *out_skb;
1093         struct skb_frag_struct *frag;
1094         int bdnum_for_frag;
1095         int frag_num;
1096         int buf_num;
1097         int size;
1098         int i;
1099
1100         size = skb_headlen(skb);
1101         buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1102
1103         frag_num = skb_shinfo(skb)->nr_frags;
1104         for (i = 0; i < frag_num; i++) {
1105                 frag = &skb_shinfo(skb)->frags[i];
1106                 size = skb_frag_size(frag);
1107                 bdnum_for_frag =
1108                         (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1109                 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
1110                         return -ENOMEM;
1111
1112                 buf_num += bdnum_for_frag;
1113         }
1114
1115         if (buf_num > ring_space(ring))
1116                 return -EBUSY;
1117
1118         *bnum = buf_num;
1119         return 0;
1120 }
1121
1122 static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
1123                                   struct hns3_enet_ring *ring)
1124 {
1125         struct sk_buff *skb = *out_skb;
1126         int buf_num;
1127
1128         /* No. of segments (plus a header) */
1129         buf_num = skb_shinfo(skb)->nr_frags + 1;
1130
1131         if (unlikely(ring_space(ring) < buf_num))
1132                 return -EBUSY;
1133
1134         *bnum = buf_num;
1135
1136         return 0;
1137 }
1138
1139 static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)
1140 {
1141         struct device *dev = ring_to_dev(ring);
1142         unsigned int i;
1143
1144         for (i = 0; i < ring->desc_num; i++) {
1145                 /* check if this is where we started */
1146                 if (ring->next_to_use == next_to_use_orig)
1147                         break;
1148
1149                 /* unmap the descriptor dma address */
1150                 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
1151                         dma_unmap_single(dev,
1152                                          ring->desc_cb[ring->next_to_use].dma,
1153                                         ring->desc_cb[ring->next_to_use].length,
1154                                         DMA_TO_DEVICE);
1155                 else if (ring->desc_cb[ring->next_to_use].length)
1156                         dma_unmap_page(dev,
1157                                        ring->desc_cb[ring->next_to_use].dma,
1158                                        ring->desc_cb[ring->next_to_use].length,
1159                                        DMA_TO_DEVICE);
1160
1161                 ring->desc_cb[ring->next_to_use].length = 0;
1162
1163                 /* rollback one */
1164                 ring_ptr_move_bw(ring, next_to_use);
1165         }
1166 }
1167
1168 netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
1169 {
1170         struct hns3_nic_priv *priv = netdev_priv(netdev);
1171         struct hns3_nic_ring_data *ring_data =
1172                 &tx_ring_data(priv, skb->queue_mapping);
1173         struct hns3_enet_ring *ring = ring_data->ring;
1174         struct netdev_queue *dev_queue;
1175         struct skb_frag_struct *frag;
1176         int next_to_use_head;
1177         int next_to_use_frag;
1178         int buf_num;
1179         int seg_num;
1180         int size;
1181         int ret;
1182         int i;
1183
1184         /* Prefetch the data used later */
1185         prefetch(skb->data);
1186
1187         switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
1188         case -EBUSY:
1189                 u64_stats_update_begin(&ring->syncp);
1190                 ring->stats.tx_busy++;
1191                 u64_stats_update_end(&ring->syncp);
1192
1193                 goto out_net_tx_busy;
1194         case -ENOMEM:
1195                 u64_stats_update_begin(&ring->syncp);
1196                 ring->stats.sw_err_cnt++;
1197                 u64_stats_update_end(&ring->syncp);
1198                 netdev_err(netdev, "no memory to xmit!\n");
1199
1200                 goto out_err_tx_ok;
1201         default:
1202                 break;
1203         }
1204
1205         /* No. of segments (plus a header) */
1206         seg_num = skb_shinfo(skb)->nr_frags + 1;
1207         /* Fill the first part */
1208         size = skb_headlen(skb);
1209
1210         next_to_use_head = ring->next_to_use;
1211
1212         ret = priv->ops.fill_desc(ring, skb, size, seg_num == 1 ? 1 : 0,
1213                                   DESC_TYPE_SKB);
1214         if (ret)
1215                 goto head_fill_err;
1216
1217         next_to_use_frag = ring->next_to_use;
1218         /* Fill the fragments */
1219         for (i = 1; i < seg_num; i++) {
1220                 frag = &skb_shinfo(skb)->frags[i - 1];
1221                 size = skb_frag_size(frag);
1222
1223                 ret = priv->ops.fill_desc(ring, frag, size,
1224                                           seg_num - 1 == i ? 1 : 0,
1225                                           DESC_TYPE_PAGE);
1226
1227                 if (ret)
1228                         goto frag_fill_err;
1229         }
1230
1231         /* Complete translate all packets */
1232         dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
1233         netdev_tx_sent_queue(dev_queue, skb->len);
1234
1235         wmb(); /* Commit all data before submit */
1236
1237         hnae3_queue_xmit(ring->tqp, buf_num);
1238
1239         return NETDEV_TX_OK;
1240
1241 frag_fill_err:
1242         hns3_clear_desc(ring, next_to_use_frag);
1243
1244 head_fill_err:
1245         hns3_clear_desc(ring, next_to_use_head);
1246
1247 out_err_tx_ok:
1248         dev_kfree_skb_any(skb);
1249         return NETDEV_TX_OK;
1250
1251 out_net_tx_busy:
1252         netif_stop_subqueue(netdev, ring_data->queue_index);
1253         smp_mb(); /* Commit all data before submit */
1254
1255         return NETDEV_TX_BUSY;
1256 }
1257
1258 static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1259 {
1260         struct hnae3_handle *h = hns3_get_handle(netdev);
1261         struct sockaddr *mac_addr = p;
1262         int ret;
1263
1264         if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1265                 return -EADDRNOTAVAIL;
1266
1267         if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
1268                 netdev_info(netdev, "already using mac address %pM\n",
1269                             mac_addr->sa_data);
1270                 return 0;
1271         }
1272
1273         ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
1274         if (ret) {
1275                 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1276                 return ret;
1277         }
1278
1279         ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1280
1281         return 0;
1282 }
1283
1284 static int hns3_nic_do_ioctl(struct net_device *netdev,
1285                              struct ifreq *ifr, int cmd)
1286 {
1287         struct hnae3_handle *h = hns3_get_handle(netdev);
1288
1289         if (!netif_running(netdev))
1290                 return -EINVAL;
1291
1292         if (!h->ae_algo->ops->do_ioctl)
1293                 return -EOPNOTSUPP;
1294
1295         return h->ae_algo->ops->do_ioctl(h, ifr, cmd);
1296 }
1297
1298 static int hns3_nic_set_features(struct net_device *netdev,
1299                                  netdev_features_t features)
1300 {
1301         netdev_features_t changed = netdev->features ^ features;
1302         struct hns3_nic_priv *priv = netdev_priv(netdev);
1303         struct hnae3_handle *h = priv->ae_handle;
1304         int ret;
1305
1306         if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
1307                 if (features & (NETIF_F_TSO | NETIF_F_TSO6))
1308                         priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1309                 else
1310                         priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1311         }
1312
1313         if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1314             h->ae_algo->ops->enable_vlan_filter) {
1315                 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1316                         h->ae_algo->ops->enable_vlan_filter(h, true);
1317                 else
1318                         h->ae_algo->ops->enable_vlan_filter(h, false);
1319         }
1320
1321         if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1322             h->ae_algo->ops->enable_hw_strip_rxvtag) {
1323                 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1324                         ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1325                 else
1326                         ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1327
1328                 if (ret)
1329                         return ret;
1330         }
1331
1332         if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {
1333                 if (features & NETIF_F_NTUPLE)
1334                         h->ae_algo->ops->enable_fd(h, true);
1335                 else
1336                         h->ae_algo->ops->enable_fd(h, false);
1337         }
1338
1339         netdev->features = features;
1340         return 0;
1341 }
1342
1343 static void hns3_nic_get_stats64(struct net_device *netdev,
1344                                  struct rtnl_link_stats64 *stats)
1345 {
1346         struct hns3_nic_priv *priv = netdev_priv(netdev);
1347         int queue_num = priv->ae_handle->kinfo.num_tqps;
1348         struct hnae3_handle *handle = priv->ae_handle;
1349         struct hns3_enet_ring *ring;
1350         unsigned int start;
1351         unsigned int idx;
1352         u64 tx_bytes = 0;
1353         u64 rx_bytes = 0;
1354         u64 tx_pkts = 0;
1355         u64 rx_pkts = 0;
1356         u64 tx_drop = 0;
1357         u64 rx_drop = 0;
1358
1359         if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1360                 return;
1361
1362         handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1363
1364         for (idx = 0; idx < queue_num; idx++) {
1365                 /* fetch the tx stats */
1366                 ring = priv->ring_data[idx].ring;
1367                 do {
1368                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1369                         tx_bytes += ring->stats.tx_bytes;
1370                         tx_pkts += ring->stats.tx_pkts;
1371                         tx_drop += ring->stats.tx_busy;
1372                         tx_drop += ring->stats.sw_err_cnt;
1373                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1374
1375                 /* fetch the rx stats */
1376                 ring = priv->ring_data[idx + queue_num].ring;
1377                 do {
1378                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1379                         rx_bytes += ring->stats.rx_bytes;
1380                         rx_pkts += ring->stats.rx_pkts;
1381                         rx_drop += ring->stats.non_vld_descs;
1382                         rx_drop += ring->stats.err_pkt_len;
1383                         rx_drop += ring->stats.l2_err;
1384                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1385         }
1386
1387         stats->tx_bytes = tx_bytes;
1388         stats->tx_packets = tx_pkts;
1389         stats->rx_bytes = rx_bytes;
1390         stats->rx_packets = rx_pkts;
1391
1392         stats->rx_errors = netdev->stats.rx_errors;
1393         stats->multicast = netdev->stats.multicast;
1394         stats->rx_length_errors = netdev->stats.rx_length_errors;
1395         stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1396         stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1397
1398         stats->tx_errors = netdev->stats.tx_errors;
1399         stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
1400         stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
1401         stats->collisions = netdev->stats.collisions;
1402         stats->rx_over_errors = netdev->stats.rx_over_errors;
1403         stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1404         stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1405         stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1406         stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1407         stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1408         stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1409         stats->tx_window_errors = netdev->stats.tx_window_errors;
1410         stats->rx_compressed = netdev->stats.rx_compressed;
1411         stats->tx_compressed = netdev->stats.tx_compressed;
1412 }
1413
1414 static int hns3_setup_tc(struct net_device *netdev, void *type_data)
1415 {
1416         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
1417         struct hnae3_handle *h = hns3_get_handle(netdev);
1418         struct hnae3_knic_private_info *kinfo = &h->kinfo;
1419         u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1420         u8 tc = mqprio_qopt->qopt.num_tc;
1421         u16 mode = mqprio_qopt->mode;
1422         u8 hw = mqprio_qopt->qopt.hw;
1423         bool if_running;
1424         int ret;
1425
1426         if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1427                mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1428                 return -EOPNOTSUPP;
1429
1430         if (tc > HNAE3_MAX_TC)
1431                 return -EINVAL;
1432
1433         if (!netdev)
1434                 return -EINVAL;
1435
1436         if_running = netif_running(netdev);
1437         if (if_running) {
1438                 hns3_nic_net_stop(netdev);
1439                 msleep(100);
1440         }
1441
1442         ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1443                 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
1444         if (ret)
1445                 goto out;
1446
1447         ret = hns3_nic_set_real_num_queue(netdev);
1448
1449 out:
1450         if (if_running)
1451                 hns3_nic_net_open(netdev);
1452
1453         return ret;
1454 }
1455
1456 static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
1457                              void *type_data)
1458 {
1459         if (type != TC_SETUP_QDISC_MQPRIO)
1460                 return -EOPNOTSUPP;
1461
1462         return hns3_setup_tc(dev, type_data);
1463 }
1464
1465 static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1466                                 __be16 proto, u16 vid)
1467 {
1468         struct hnae3_handle *h = hns3_get_handle(netdev);
1469         struct hns3_nic_priv *priv = netdev_priv(netdev);
1470         int ret = -EIO;
1471
1472         if (h->ae_algo->ops->set_vlan_filter)
1473                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1474
1475         if (!ret)
1476                 set_bit(vid, priv->active_vlans);
1477
1478         return ret;
1479 }
1480
1481 static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1482                                  __be16 proto, u16 vid)
1483 {
1484         struct hnae3_handle *h = hns3_get_handle(netdev);
1485         struct hns3_nic_priv *priv = netdev_priv(netdev);
1486         int ret = -EIO;
1487
1488         if (h->ae_algo->ops->set_vlan_filter)
1489                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1490
1491         if (!ret)
1492                 clear_bit(vid, priv->active_vlans);
1493
1494         return ret;
1495 }
1496
1497 static void hns3_restore_vlan(struct net_device *netdev)
1498 {
1499         struct hns3_nic_priv *priv = netdev_priv(netdev);
1500         u16 vid;
1501         int ret;
1502
1503         for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
1504                 ret = hns3_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
1505                 if (ret)
1506                         netdev_warn(netdev, "Restore vlan: %d filter, ret:%d\n",
1507                                     vid, ret);
1508         }
1509 }
1510
1511 static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1512                                 u8 qos, __be16 vlan_proto)
1513 {
1514         struct hnae3_handle *h = hns3_get_handle(netdev);
1515         int ret = -EIO;
1516
1517         if (h->ae_algo->ops->set_vf_vlan_filter)
1518                 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1519                                                    qos, vlan_proto);
1520
1521         return ret;
1522 }
1523
1524 static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1525 {
1526         struct hnae3_handle *h = hns3_get_handle(netdev);
1527         bool if_running = netif_running(netdev);
1528         int ret;
1529
1530         if (!h->ae_algo->ops->set_mtu)
1531                 return -EOPNOTSUPP;
1532
1533         /* if this was called with netdev up then bring netdevice down */
1534         if (if_running) {
1535                 (void)hns3_nic_net_stop(netdev);
1536                 msleep(100);
1537         }
1538
1539         ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1540         if (ret)
1541                 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1542                            ret);
1543         else
1544                 netdev->mtu = new_mtu;
1545
1546         /* if the netdev was running earlier, bring it up again */
1547         if (if_running && hns3_nic_net_open(netdev))
1548                 ret = -EINVAL;
1549
1550         return ret;
1551 }
1552
1553 static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1554 {
1555         struct hns3_nic_priv *priv = netdev_priv(ndev);
1556         struct hns3_enet_ring *tx_ring = NULL;
1557         int timeout_queue = 0;
1558         int hw_head, hw_tail;
1559         int i;
1560
1561         /* Find the stopped queue the same way the stack does */
1562         for (i = 0; i < ndev->real_num_tx_queues; i++) {
1563                 struct netdev_queue *q;
1564                 unsigned long trans_start;
1565
1566                 q = netdev_get_tx_queue(ndev, i);
1567                 trans_start = q->trans_start;
1568                 if (netif_xmit_stopped(q) &&
1569                     time_after(jiffies,
1570                                (trans_start + ndev->watchdog_timeo))) {
1571                         timeout_queue = i;
1572                         break;
1573                 }
1574         }
1575
1576         if (i == ndev->num_tx_queues) {
1577                 netdev_info(ndev,
1578                             "no netdev TX timeout queue found, timeout count: %llu\n",
1579                             priv->tx_timeout_count);
1580                 return false;
1581         }
1582
1583         tx_ring = priv->ring_data[timeout_queue].ring;
1584
1585         hw_head = readl_relaxed(tx_ring->tqp->io_base +
1586                                 HNS3_RING_TX_RING_HEAD_REG);
1587         hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1588                                 HNS3_RING_TX_RING_TAIL_REG);
1589         netdev_info(ndev,
1590                     "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1591                     priv->tx_timeout_count,
1592                     timeout_queue,
1593                     tx_ring->next_to_use,
1594                     tx_ring->next_to_clean,
1595                     hw_head,
1596                     hw_tail,
1597                     readl(tx_ring->tqp_vector->mask_addr));
1598
1599         return true;
1600 }
1601
1602 static void hns3_nic_net_timeout(struct net_device *ndev)
1603 {
1604         struct hns3_nic_priv *priv = netdev_priv(ndev);
1605         struct hnae3_handle *h = priv->ae_handle;
1606
1607         if (!hns3_get_tx_timeo_queue_info(ndev))
1608                 return;
1609
1610         priv->tx_timeout_count++;
1611
1612         if (time_before(jiffies, (h->last_reset_time + ndev->watchdog_timeo)))
1613                 return;
1614
1615         /* request the reset */
1616         if (h->ae_algo->ops->reset_event)
1617                 h->ae_algo->ops->reset_event(h->pdev, h);
1618 }
1619
1620 static const struct net_device_ops hns3_nic_netdev_ops = {
1621         .ndo_open               = hns3_nic_net_open,
1622         .ndo_stop               = hns3_nic_net_stop,
1623         .ndo_start_xmit         = hns3_nic_net_xmit,
1624         .ndo_tx_timeout         = hns3_nic_net_timeout,
1625         .ndo_set_mac_address    = hns3_nic_net_set_mac_address,
1626         .ndo_do_ioctl           = hns3_nic_do_ioctl,
1627         .ndo_change_mtu         = hns3_nic_change_mtu,
1628         .ndo_set_features       = hns3_nic_set_features,
1629         .ndo_get_stats64        = hns3_nic_get_stats64,
1630         .ndo_setup_tc           = hns3_nic_setup_tc,
1631         .ndo_set_rx_mode        = hns3_nic_set_rx_mode,
1632         .ndo_vlan_rx_add_vid    = hns3_vlan_rx_add_vid,
1633         .ndo_vlan_rx_kill_vid   = hns3_vlan_rx_kill_vid,
1634         .ndo_set_vf_vlan        = hns3_ndo_set_vf_vlan,
1635 };
1636
1637 static bool hns3_is_phys_func(struct pci_dev *pdev)
1638 {
1639         u32 dev_id = pdev->device;
1640
1641         switch (dev_id) {
1642         case HNAE3_DEV_ID_GE:
1643         case HNAE3_DEV_ID_25GE:
1644         case HNAE3_DEV_ID_25GE_RDMA:
1645         case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
1646         case HNAE3_DEV_ID_50GE_RDMA:
1647         case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
1648         case HNAE3_DEV_ID_100G_RDMA_MACSEC:
1649                 return true;
1650         case HNAE3_DEV_ID_100G_VF:
1651         case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF:
1652                 return false;
1653         default:
1654                 dev_warn(&pdev->dev, "un-recognized pci device-id %d",
1655                          dev_id);
1656         }
1657
1658         return false;
1659 }
1660
1661 static void hns3_disable_sriov(struct pci_dev *pdev)
1662 {
1663         /* If our VFs are assigned we cannot shut down SR-IOV
1664          * without causing issues, so just leave the hardware
1665          * available but disabled
1666          */
1667         if (pci_vfs_assigned(pdev)) {
1668                 dev_warn(&pdev->dev,
1669                          "disabling driver while VFs are assigned\n");
1670                 return;
1671         }
1672
1673         pci_disable_sriov(pdev);
1674 }
1675
1676 static void hns3_get_dev_capability(struct pci_dev *pdev,
1677                                     struct hnae3_ae_dev *ae_dev)
1678 {
1679         if (pdev->revision >= 0x21)
1680                 hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_FD_B, 1);
1681 }
1682
1683 /* hns3_probe - Device initialization routine
1684  * @pdev: PCI device information struct
1685  * @ent: entry in hns3_pci_tbl
1686  *
1687  * hns3_probe initializes a PF identified by a pci_dev structure.
1688  * The OS initialization, configuring of the PF private structure,
1689  * and a hardware reset occur.
1690  *
1691  * Returns 0 on success, negative on failure
1692  */
1693 static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1694 {
1695         struct hnae3_ae_dev *ae_dev;
1696         int ret;
1697
1698         ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1699                               GFP_KERNEL);
1700         if (!ae_dev) {
1701                 ret = -ENOMEM;
1702                 return ret;
1703         }
1704
1705         ae_dev->pdev = pdev;
1706         ae_dev->flag = ent->driver_data;
1707         ae_dev->dev_type = HNAE3_DEV_KNIC;
1708         ae_dev->reset_type = HNAE3_NONE_RESET;
1709         hns3_get_dev_capability(pdev, ae_dev);
1710         pci_set_drvdata(pdev, ae_dev);
1711
1712         hnae3_register_ae_dev(ae_dev);
1713
1714         return 0;
1715 }
1716
1717 /* hns3_remove - Device removal routine
1718  * @pdev: PCI device information struct
1719  */
1720 static void hns3_remove(struct pci_dev *pdev)
1721 {
1722         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1723
1724         if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
1725                 hns3_disable_sriov(pdev);
1726
1727         hnae3_unregister_ae_dev(ae_dev);
1728 }
1729
1730 /**
1731  * hns3_pci_sriov_configure
1732  * @pdev: pointer to a pci_dev structure
1733  * @num_vfs: number of VFs to allocate
1734  *
1735  * Enable or change the number of VFs. Called when the user updates the number
1736  * of VFs in sysfs.
1737  **/
1738 static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
1739 {
1740         int ret;
1741
1742         if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
1743                 dev_warn(&pdev->dev, "Can not config SRIOV\n");
1744                 return -EINVAL;
1745         }
1746
1747         if (num_vfs) {
1748                 ret = pci_enable_sriov(pdev, num_vfs);
1749                 if (ret)
1750                         dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
1751                 else
1752                         return num_vfs;
1753         } else if (!pci_vfs_assigned(pdev)) {
1754                 pci_disable_sriov(pdev);
1755         } else {
1756                 dev_warn(&pdev->dev,
1757                          "Unable to free VFs because some are assigned to VMs.\n");
1758         }
1759
1760         return 0;
1761 }
1762
1763 static void hns3_shutdown(struct pci_dev *pdev)
1764 {
1765         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1766
1767         hnae3_unregister_ae_dev(ae_dev);
1768         devm_kfree(&pdev->dev, ae_dev);
1769         pci_set_drvdata(pdev, NULL);
1770
1771         if (system_state == SYSTEM_POWER_OFF)
1772                 pci_set_power_state(pdev, PCI_D3hot);
1773 }
1774
1775 static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,
1776                                             pci_channel_state_t state)
1777 {
1778         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1779         pci_ers_result_t ret;
1780
1781         dev_info(&pdev->dev, "PCI error detected, state(=%d)!!\n", state);
1782
1783         if (state == pci_channel_io_perm_failure)
1784                 return PCI_ERS_RESULT_DISCONNECT;
1785
1786         if (!ae_dev) {
1787                 dev_err(&pdev->dev,
1788                         "Can't recover - error happened during device init\n");
1789                 return PCI_ERS_RESULT_NONE;
1790         }
1791
1792         if (ae_dev->ops->process_hw_error)
1793                 ret = ae_dev->ops->process_hw_error(ae_dev);
1794         else
1795                 return PCI_ERS_RESULT_NONE;
1796
1797         return ret;
1798 }
1799
1800 static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev)
1801 {
1802         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1803         struct device *dev = &pdev->dev;
1804
1805         dev_info(dev, "requesting reset due to PCI error\n");
1806
1807         /* request the reset */
1808         if (ae_dev->ops->reset_event) {
1809                 ae_dev->ops->reset_event(pdev, NULL);
1810                 return PCI_ERS_RESULT_RECOVERED;
1811         }
1812
1813         return PCI_ERS_RESULT_DISCONNECT;
1814 }
1815
1816 static const struct pci_error_handlers hns3_err_handler = {
1817         .error_detected = hns3_error_detected,
1818         .slot_reset     = hns3_slot_reset,
1819 };
1820
1821 static struct pci_driver hns3_driver = {
1822         .name     = hns3_driver_name,
1823         .id_table = hns3_pci_tbl,
1824         .probe    = hns3_probe,
1825         .remove   = hns3_remove,
1826         .shutdown = hns3_shutdown,
1827         .sriov_configure = hns3_pci_sriov_configure,
1828         .err_handler    = &hns3_err_handler,
1829 };
1830
1831 /* set default feature to hns3 */
1832 static void hns3_set_default_feature(struct net_device *netdev)
1833 {
1834         struct hnae3_handle *h = hns3_get_handle(netdev);
1835         struct pci_dev *pdev = h->pdev;
1836
1837         netdev->priv_flags |= IFF_UNICAST_FLT;
1838
1839         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1840                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1841                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1842                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1843                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1844
1845         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1846
1847         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1848
1849         netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1850                 NETIF_F_HW_VLAN_CTAG_FILTER |
1851                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
1852                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1853                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1854                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1855                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1856
1857         netdev->vlan_features |=
1858                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1859                 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1860                 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1861                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1862                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1863
1864         netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1865                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
1866                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1867                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1868                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1869                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1870
1871         if (pdev->revision >= 0x21) {
1872                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1873
1874                 if (!(h->flags & HNAE3_SUPPORT_VF)) {
1875                         netdev->hw_features |= NETIF_F_NTUPLE;
1876                         netdev->features |= NETIF_F_NTUPLE;
1877                 }
1878         }
1879 }
1880
1881 static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1882                              struct hns3_desc_cb *cb)
1883 {
1884         unsigned int order = hnae3_page_order(ring);
1885         struct page *p;
1886
1887         p = dev_alloc_pages(order);
1888         if (!p)
1889                 return -ENOMEM;
1890
1891         cb->priv = p;
1892         cb->page_offset = 0;
1893         cb->reuse_flag = 0;
1894         cb->buf  = page_address(p);
1895         cb->length = hnae3_page_size(ring);
1896         cb->type = DESC_TYPE_PAGE;
1897
1898         return 0;
1899 }
1900
1901 static void hns3_free_buffer(struct hns3_enet_ring *ring,
1902                              struct hns3_desc_cb *cb)
1903 {
1904         if (cb->type == DESC_TYPE_SKB)
1905                 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1906         else if (!HNAE3_IS_TX_RING(ring))
1907                 put_page((struct page *)cb->priv);
1908         memset(cb, 0, sizeof(*cb));
1909 }
1910
1911 static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1912 {
1913         cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1914                                cb->length, ring_to_dma_dir(ring));
1915
1916         if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))
1917                 return -EIO;
1918
1919         return 0;
1920 }
1921
1922 static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1923                               struct hns3_desc_cb *cb)
1924 {
1925         if (cb->type == DESC_TYPE_SKB)
1926                 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1927                                  ring_to_dma_dir(ring));
1928         else if (cb->length)
1929                 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1930                                ring_to_dma_dir(ring));
1931 }
1932
1933 static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1934 {
1935         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1936         ring->desc[i].addr = 0;
1937 }
1938
1939 static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1940 {
1941         struct hns3_desc_cb *cb = &ring->desc_cb[i];
1942
1943         if (!ring->desc_cb[i].dma)
1944                 return;
1945
1946         hns3_buffer_detach(ring, i);
1947         hns3_free_buffer(ring, cb);
1948 }
1949
1950 static void hns3_free_buffers(struct hns3_enet_ring *ring)
1951 {
1952         int i;
1953
1954         for (i = 0; i < ring->desc_num; i++)
1955                 hns3_free_buffer_detach(ring, i);
1956 }
1957
1958 /* free desc along with its attached buffer */
1959 static void hns3_free_desc(struct hns3_enet_ring *ring)
1960 {
1961         int size = ring->desc_num * sizeof(ring->desc[0]);
1962
1963         hns3_free_buffers(ring);
1964
1965         if (ring->desc) {
1966                 dma_free_coherent(ring_to_dev(ring), size,
1967                                   ring->desc, ring->desc_dma_addr);
1968                 ring->desc = NULL;
1969         }
1970 }
1971
1972 static int hns3_alloc_desc(struct hns3_enet_ring *ring)
1973 {
1974         int size = ring->desc_num * sizeof(ring->desc[0]);
1975
1976         ring->desc = dma_zalloc_coherent(ring_to_dev(ring), size,
1977                                          &ring->desc_dma_addr,
1978                                          GFP_KERNEL);
1979         if (!ring->desc)
1980                 return -ENOMEM;
1981
1982         return 0;
1983 }
1984
1985 static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
1986                                    struct hns3_desc_cb *cb)
1987 {
1988         int ret;
1989
1990         ret = hns3_alloc_buffer(ring, cb);
1991         if (ret)
1992                 goto out;
1993
1994         ret = hns3_map_buffer(ring, cb);
1995         if (ret)
1996                 goto out_with_buf;
1997
1998         return 0;
1999
2000 out_with_buf:
2001         hns3_free_buffer(ring, cb);
2002 out:
2003         return ret;
2004 }
2005
2006 static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
2007 {
2008         int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
2009
2010         if (ret)
2011                 return ret;
2012
2013         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2014
2015         return 0;
2016 }
2017
2018 /* Allocate memory for raw pkg, and map with dma */
2019 static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
2020 {
2021         int i, j, ret;
2022
2023         for (i = 0; i < ring->desc_num; i++) {
2024                 ret = hns3_alloc_buffer_attach(ring, i);
2025                 if (ret)
2026                         goto out_buffer_fail;
2027         }
2028
2029         return 0;
2030
2031 out_buffer_fail:
2032         for (j = i - 1; j >= 0; j--)
2033                 hns3_free_buffer_detach(ring, j);
2034         return ret;
2035 }
2036
2037 /* detach a in-used buffer and replace with a reserved one  */
2038 static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
2039                                 struct hns3_desc_cb *res_cb)
2040 {
2041         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
2042         ring->desc_cb[i] = *res_cb;
2043         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2044         ring->desc[i].rx.bd_base_info = 0;
2045 }
2046
2047 static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
2048 {
2049         ring->desc_cb[i].reuse_flag = 0;
2050         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
2051                 + ring->desc_cb[i].page_offset);
2052         ring->desc[i].rx.bd_base_info = 0;
2053 }
2054
2055 static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
2056                                       int *pkts)
2057 {
2058         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
2059
2060         (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
2061         (*bytes) += desc_cb->length;
2062         /* desc_cb will be cleaned, after hnae3_free_buffer_detach*/
2063         hns3_free_buffer_detach(ring, ring->next_to_clean);
2064
2065         ring_ptr_move_fw(ring, next_to_clean);
2066 }
2067
2068 static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
2069 {
2070         int u = ring->next_to_use;
2071         int c = ring->next_to_clean;
2072
2073         if (unlikely(h > ring->desc_num))
2074                 return 0;
2075
2076         return u > c ? (h > c && h <= u) : (h > c || h <= u);
2077 }
2078
2079 void hns3_clean_tx_ring(struct hns3_enet_ring *ring)
2080 {
2081         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2082         struct hns3_nic_priv *priv = netdev_priv(netdev);
2083         struct netdev_queue *dev_queue;
2084         int bytes, pkts;
2085         int head;
2086
2087         head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
2088         rmb(); /* Make sure head is ready before touch any data */
2089
2090         if (is_ring_empty(ring) || head == ring->next_to_clean)
2091                 return; /* no data to poll */
2092
2093         if (unlikely(!is_valid_clean_head(ring, head))) {
2094                 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
2095                            ring->next_to_use, ring->next_to_clean);
2096
2097                 u64_stats_update_begin(&ring->syncp);
2098                 ring->stats.io_err_cnt++;
2099                 u64_stats_update_end(&ring->syncp);
2100                 return;
2101         }
2102
2103         bytes = 0;
2104         pkts = 0;
2105         while (head != ring->next_to_clean) {
2106                 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
2107                 /* Issue prefetch for next Tx descriptor */
2108                 prefetch(&ring->desc_cb[ring->next_to_clean]);
2109         }
2110
2111         ring->tqp_vector->tx_group.total_bytes += bytes;
2112         ring->tqp_vector->tx_group.total_packets += pkts;
2113
2114         u64_stats_update_begin(&ring->syncp);
2115         ring->stats.tx_bytes += bytes;
2116         ring->stats.tx_pkts += pkts;
2117         u64_stats_update_end(&ring->syncp);
2118
2119         dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
2120         netdev_tx_completed_queue(dev_queue, pkts, bytes);
2121
2122         if (unlikely(pkts && netif_carrier_ok(netdev) &&
2123                      (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
2124                 /* Make sure that anybody stopping the queue after this
2125                  * sees the new next_to_clean.
2126                  */
2127                 smp_mb();
2128                 if (netif_tx_queue_stopped(dev_queue) &&
2129                     !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
2130                         netif_tx_wake_queue(dev_queue);
2131                         ring->stats.restart_queue++;
2132                 }
2133         }
2134 }
2135
2136 static int hns3_desc_unused(struct hns3_enet_ring *ring)
2137 {
2138         int ntc = ring->next_to_clean;
2139         int ntu = ring->next_to_use;
2140
2141         return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
2142 }
2143
2144 static void
2145 hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
2146 {
2147         struct hns3_desc_cb *desc_cb;
2148         struct hns3_desc_cb res_cbs;
2149         int i, ret;
2150
2151         for (i = 0; i < cleand_count; i++) {
2152                 desc_cb = &ring->desc_cb[ring->next_to_use];
2153                 if (desc_cb->reuse_flag) {
2154                         u64_stats_update_begin(&ring->syncp);
2155                         ring->stats.reuse_pg_cnt++;
2156                         u64_stats_update_end(&ring->syncp);
2157
2158                         hns3_reuse_buffer(ring, ring->next_to_use);
2159                 } else {
2160                         ret = hns3_reserve_buffer_map(ring, &res_cbs);
2161                         if (ret) {
2162                                 u64_stats_update_begin(&ring->syncp);
2163                                 ring->stats.sw_err_cnt++;
2164                                 u64_stats_update_end(&ring->syncp);
2165
2166                                 netdev_err(ring->tqp->handle->kinfo.netdev,
2167                                            "hnae reserve buffer map failed.\n");
2168                                 break;
2169                         }
2170                         hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
2171                 }
2172
2173                 ring_ptr_move_fw(ring, next_to_use);
2174         }
2175
2176         wmb(); /* Make all data has been write before submit */
2177         writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
2178 }
2179
2180 static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2181                                 struct hns3_enet_ring *ring, int pull_len,
2182                                 struct hns3_desc_cb *desc_cb)
2183 {
2184         struct hns3_desc *desc;
2185         u32 truesize;
2186         int size;
2187         int last_offset;
2188         bool twobufs;
2189
2190         twobufs = ((PAGE_SIZE < 8192) &&
2191                 hnae3_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
2192
2193         desc = &ring->desc[ring->next_to_clean];
2194         size = le16_to_cpu(desc->rx.size);
2195
2196         truesize = hnae3_buf_size(ring);
2197
2198         if (!twobufs)
2199                 last_offset = hnae3_page_size(ring) - hnae3_buf_size(ring);
2200
2201         skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
2202                         size - pull_len, truesize);
2203
2204          /* Avoid re-using remote pages,flag default unreuse */
2205         if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
2206                 return;
2207
2208         if (twobufs) {
2209                 /* If we are only owner of page we can reuse it */
2210                 if (likely(page_count(desc_cb->priv) == 1)) {
2211                         /* Flip page offset to other buffer */
2212                         desc_cb->page_offset ^= truesize;
2213
2214                         desc_cb->reuse_flag = 1;
2215                         /* bump ref count on page before it is given*/
2216                         get_page(desc_cb->priv);
2217                 }
2218                 return;
2219         }
2220
2221         /* Move offset up to the next cache line */
2222         desc_cb->page_offset += truesize;
2223
2224         if (desc_cb->page_offset <= last_offset) {
2225                 desc_cb->reuse_flag = 1;
2226                 /* Bump ref count on page before it is given*/
2227                 get_page(desc_cb->priv);
2228         }
2229 }
2230
2231 static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2232                              struct hns3_desc *desc)
2233 {
2234         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2235         int l3_type, l4_type;
2236         u32 bd_base_info;
2237         int ol4_type;
2238         u32 l234info;
2239
2240         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2241         l234info = le32_to_cpu(desc->rx.l234_info);
2242
2243         skb->ip_summed = CHECKSUM_NONE;
2244
2245         skb_checksum_none_assert(skb);
2246
2247         if (!(netdev->features & NETIF_F_RXCSUM))
2248                 return;
2249
2250         /* check if hardware has done checksum */
2251         if (!hnae3_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
2252                 return;
2253
2254         if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L3E_B) ||
2255                      hnae3_get_bit(l234info, HNS3_RXD_L4E_B) ||
2256                      hnae3_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2257                      hnae3_get_bit(l234info, HNS3_RXD_OL4E_B))) {
2258                 u64_stats_update_begin(&ring->syncp);
2259                 ring->stats.l3l4_csum_err++;
2260                 u64_stats_update_end(&ring->syncp);
2261
2262                 return;
2263         }
2264
2265         l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
2266                                   HNS3_RXD_L3ID_S);
2267         l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,
2268                                   HNS3_RXD_L4ID_S);
2269
2270         ol4_type = hnae3_get_field(l234info, HNS3_RXD_OL4ID_M,
2271                                    HNS3_RXD_OL4ID_S);
2272         switch (ol4_type) {
2273         case HNS3_OL4_TYPE_MAC_IN_UDP:
2274         case HNS3_OL4_TYPE_NVGRE:
2275                 skb->csum_level = 1;
2276                 /* fall through */
2277         case HNS3_OL4_TYPE_NO_TUN:
2278                 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2279                 if ((l3_type == HNS3_L3_TYPE_IPV4 ||
2280                      l3_type == HNS3_L3_TYPE_IPV6) &&
2281                     (l4_type == HNS3_L4_TYPE_UDP ||
2282                      l4_type == HNS3_L4_TYPE_TCP ||
2283                      l4_type == HNS3_L4_TYPE_SCTP))
2284                         skb->ip_summed = CHECKSUM_UNNECESSARY;
2285                 break;
2286         default:
2287                 break;
2288         }
2289 }
2290
2291 static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2292 {
2293         napi_gro_receive(&ring->tqp_vector->napi, skb);
2294 }
2295
2296 static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
2297                                 struct hns3_desc *desc, u32 l234info,
2298                                 u16 *vlan_tag)
2299 {
2300         struct pci_dev *pdev = ring->tqp->handle->pdev;
2301
2302         if (pdev->revision == 0x20) {
2303                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2304                 if (!(*vlan_tag & VLAN_VID_MASK))
2305                         *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2306
2307                 return (*vlan_tag != 0);
2308         }
2309
2310 #define HNS3_STRP_OUTER_VLAN    0x1
2311 #define HNS3_STRP_INNER_VLAN    0x2
2312
2313         switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
2314                                 HNS3_RXD_STRP_TAGP_S)) {
2315         case HNS3_STRP_OUTER_VLAN:
2316                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2317                 return true;
2318         case HNS3_STRP_INNER_VLAN:
2319                 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2320                 return true;
2321         default:
2322                 return false;
2323         }
2324 }
2325
2326 static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring,
2327                                      struct sk_buff *skb)
2328 {
2329         struct hns3_desc *desc = &ring->desc[ring->next_to_clean];
2330         struct hnae3_handle *handle = ring->tqp->handle;
2331         enum pkt_hash_types rss_type;
2332
2333         if (le32_to_cpu(desc->rx.rss_hash))
2334                 rss_type = handle->kinfo.rss_type;
2335         else
2336                 rss_type = PKT_HASH_TYPE_NONE;
2337
2338         skb_set_hash(skb, le32_to_cpu(desc->rx.rss_hash), rss_type);
2339 }
2340
2341 static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
2342                              struct sk_buff **out_skb, int *out_bnum)
2343 {
2344         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2345         struct hns3_desc_cb *desc_cb;
2346         struct hns3_desc *desc;
2347         struct sk_buff *skb;
2348         unsigned char *va;
2349         u32 bd_base_info;
2350         int pull_len;
2351         u32 l234info;
2352         int length;
2353         int bnum;
2354
2355         desc = &ring->desc[ring->next_to_clean];
2356         desc_cb = &ring->desc_cb[ring->next_to_clean];
2357
2358         prefetch(desc);
2359
2360         length = le16_to_cpu(desc->rx.size);
2361         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2362
2363         /* Check valid BD */
2364         if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B)))
2365                 return -EFAULT;
2366
2367         va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
2368
2369         /* Prefetch first cache line of first page
2370          * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2371          * line size is 64B so need to prefetch twice to make it 128B. But in
2372          * actual we can have greater size of caches with 128B Level 1 cache
2373          * lines. In such a case, single fetch would suffice to cache in the
2374          * relevant part of the header.
2375          */
2376         prefetch(va);
2377 #if L1_CACHE_BYTES < 128
2378         prefetch(va + L1_CACHE_BYTES);
2379 #endif
2380
2381         skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2382                                         HNS3_RX_HEAD_SIZE);
2383         if (unlikely(!skb)) {
2384                 netdev_err(netdev, "alloc rx skb fail\n");
2385
2386                 u64_stats_update_begin(&ring->syncp);
2387                 ring->stats.sw_err_cnt++;
2388                 u64_stats_update_end(&ring->syncp);
2389
2390                 return -ENOMEM;
2391         }
2392
2393         prefetchw(skb->data);
2394
2395         bnum = 1;
2396         if (length <= HNS3_RX_HEAD_SIZE) {
2397                 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2398
2399                 /* We can reuse buffer as-is, just make sure it is local */
2400                 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2401                         desc_cb->reuse_flag = 1;
2402                 else /* This page cannot be reused so discard it */
2403                         put_page(desc_cb->priv);
2404
2405                 ring_ptr_move_fw(ring, next_to_clean);
2406         } else {
2407                 u64_stats_update_begin(&ring->syncp);
2408                 ring->stats.seg_pkt_cnt++;
2409                 u64_stats_update_end(&ring->syncp);
2410
2411                 pull_len = eth_get_headlen(va, HNS3_RX_HEAD_SIZE);
2412
2413                 memcpy(__skb_put(skb, pull_len), va,
2414                        ALIGN(pull_len, sizeof(long)));
2415
2416                 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
2417                 ring_ptr_move_fw(ring, next_to_clean);
2418
2419                 while (!hnae3_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2420                         desc = &ring->desc[ring->next_to_clean];
2421                         desc_cb = &ring->desc_cb[ring->next_to_clean];
2422                         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2423                         hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
2424                         ring_ptr_move_fw(ring, next_to_clean);
2425                         bnum++;
2426                 }
2427         }
2428
2429         *out_bnum = bnum;
2430
2431         l234info = le32_to_cpu(desc->rx.l234_info);
2432
2433         /* Based on hw strategy, the tag offloaded will be stored at
2434          * ot_vlan_tag in two layer tag case, and stored at vlan_tag
2435          * in one layer tag case.
2436          */
2437         if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
2438                 u16 vlan_tag;
2439
2440                 if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))
2441                         __vlan_hwaccel_put_tag(skb,
2442                                                htons(ETH_P_8021Q),
2443                                                vlan_tag);
2444         }
2445
2446         if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
2447                 u64_stats_update_begin(&ring->syncp);
2448                 ring->stats.non_vld_descs++;
2449                 u64_stats_update_end(&ring->syncp);
2450
2451                 dev_kfree_skb_any(skb);
2452                 return -EINVAL;
2453         }
2454
2455         if (unlikely((!desc->rx.pkt_len) ||
2456                      hnae3_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
2457                 u64_stats_update_begin(&ring->syncp);
2458                 ring->stats.err_pkt_len++;
2459                 u64_stats_update_end(&ring->syncp);
2460
2461                 dev_kfree_skb_any(skb);
2462                 return -EFAULT;
2463         }
2464
2465         if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L2E_B))) {
2466                 u64_stats_update_begin(&ring->syncp);
2467                 ring->stats.l2_err++;
2468                 u64_stats_update_end(&ring->syncp);
2469
2470                 dev_kfree_skb_any(skb);
2471                 return -EFAULT;
2472         }
2473
2474         u64_stats_update_begin(&ring->syncp);
2475         ring->stats.rx_pkts++;
2476         ring->stats.rx_bytes += skb->len;
2477         u64_stats_update_end(&ring->syncp);
2478
2479         ring->tqp_vector->rx_group.total_bytes += skb->len;
2480
2481         hns3_rx_checksum(ring, skb, desc);
2482         hns3_set_rx_skb_rss_type(ring, skb);
2483
2484         return 0;
2485 }
2486
2487 int hns3_clean_rx_ring(
2488                 struct hns3_enet_ring *ring, int budget,
2489                 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
2490 {
2491 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2492         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2493         int recv_pkts, recv_bds, clean_count, err;
2494         int unused_count = hns3_desc_unused(ring);
2495         struct sk_buff *skb = NULL;
2496         int num, bnum = 0;
2497
2498         num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2499         rmb(); /* Make sure num taken effect before the other data is touched */
2500
2501         recv_pkts = 0, recv_bds = 0, clean_count = 0;
2502         num -= unused_count;
2503
2504         while (recv_pkts < budget && recv_bds < num) {
2505                 /* Reuse or realloc buffers */
2506                 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2507                         hns3_nic_alloc_rx_buffers(ring,
2508                                                   clean_count + unused_count);
2509                         clean_count = 0;
2510                         unused_count = hns3_desc_unused(ring);
2511                 }
2512
2513                 /* Poll one pkt */
2514                 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2515                 if (unlikely(!skb)) /* This fault cannot be repaired */
2516                         goto out;
2517
2518                 recv_bds += bnum;
2519                 clean_count += bnum;
2520                 if (unlikely(err)) {  /* Do jump the err */
2521                         recv_pkts++;
2522                         continue;
2523                 }
2524
2525                 /* Do update ip stack process */
2526                 skb->protocol = eth_type_trans(skb, netdev);
2527                 rx_fn(ring, skb);
2528
2529                 recv_pkts++;
2530         }
2531
2532 out:
2533         /* Make all data has been write before submit */
2534         if (clean_count + unused_count > 0)
2535                 hns3_nic_alloc_rx_buffers(ring,
2536                                           clean_count + unused_count);
2537
2538         return recv_pkts;
2539 }
2540
2541 static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2542 {
2543         struct hns3_enet_tqp_vector *tqp_vector =
2544                                         ring_group->ring->tqp_vector;
2545         enum hns3_flow_level_range new_flow_level;
2546         int packets_per_msecs;
2547         int bytes_per_msecs;
2548         u32 time_passed_ms;
2549         u16 new_int_gl;
2550
2551         if (!ring_group->coal.int_gl || !tqp_vector->last_jiffies)
2552                 return false;
2553
2554         if (ring_group->total_packets == 0) {
2555                 ring_group->coal.int_gl = HNS3_INT_GL_50K;
2556                 ring_group->coal.flow_level = HNS3_FLOW_LOW;
2557                 return true;
2558         }
2559
2560         /* Simple throttlerate management
2561          * 0-10MB/s   lower     (50000 ints/s)
2562          * 10-20MB/s   middle    (20000 ints/s)
2563          * 20-1249MB/s high      (18000 ints/s)
2564          * > 40000pps  ultra     (8000 ints/s)
2565          */
2566         new_flow_level = ring_group->coal.flow_level;
2567         new_int_gl = ring_group->coal.int_gl;
2568         time_passed_ms =
2569                 jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);
2570
2571         if (!time_passed_ms)
2572                 return false;
2573
2574         do_div(ring_group->total_packets, time_passed_ms);
2575         packets_per_msecs = ring_group->total_packets;
2576
2577         do_div(ring_group->total_bytes, time_passed_ms);
2578         bytes_per_msecs = ring_group->total_bytes;
2579
2580 #define HNS3_RX_LOW_BYTE_RATE 10000
2581 #define HNS3_RX_MID_BYTE_RATE 20000
2582
2583         switch (new_flow_level) {
2584         case HNS3_FLOW_LOW:
2585                 if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE)
2586                         new_flow_level = HNS3_FLOW_MID;
2587                 break;
2588         case HNS3_FLOW_MID:
2589                 if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE)
2590                         new_flow_level = HNS3_FLOW_HIGH;
2591                 else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE)
2592                         new_flow_level = HNS3_FLOW_LOW;
2593                 break;
2594         case HNS3_FLOW_HIGH:
2595         case HNS3_FLOW_ULTRA:
2596         default:
2597                 if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE)
2598                         new_flow_level = HNS3_FLOW_MID;
2599                 break;
2600         }
2601
2602 #define HNS3_RX_ULTRA_PACKET_RATE 40
2603
2604         if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE &&
2605             &tqp_vector->rx_group == ring_group)
2606                 new_flow_level = HNS3_FLOW_ULTRA;
2607
2608         switch (new_flow_level) {
2609         case HNS3_FLOW_LOW:
2610                 new_int_gl = HNS3_INT_GL_50K;
2611                 break;
2612         case HNS3_FLOW_MID:
2613                 new_int_gl = HNS3_INT_GL_20K;
2614                 break;
2615         case HNS3_FLOW_HIGH:
2616                 new_int_gl = HNS3_INT_GL_18K;
2617                 break;
2618         case HNS3_FLOW_ULTRA:
2619                 new_int_gl = HNS3_INT_GL_8K;
2620                 break;
2621         default:
2622                 break;
2623         }
2624
2625         ring_group->total_bytes = 0;
2626         ring_group->total_packets = 0;
2627         ring_group->coal.flow_level = new_flow_level;
2628         if (new_int_gl != ring_group->coal.int_gl) {
2629                 ring_group->coal.int_gl = new_int_gl;
2630                 return true;
2631         }
2632         return false;
2633 }
2634
2635 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2636 {
2637         struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
2638         struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
2639         bool rx_update, tx_update;
2640
2641         if (tqp_vector->int_adapt_down > 0) {
2642                 tqp_vector->int_adapt_down--;
2643                 return;
2644         }
2645
2646         if (rx_group->coal.gl_adapt_enable) {
2647                 rx_update = hns3_get_new_int_gl(rx_group);
2648                 if (rx_update)
2649                         hns3_set_vector_coalesce_rx_gl(tqp_vector,
2650                                                        rx_group->coal.int_gl);
2651         }
2652
2653         if (tx_group->coal.gl_adapt_enable) {
2654                 tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
2655                 if (tx_update)
2656                         hns3_set_vector_coalesce_tx_gl(tqp_vector,
2657                                                        tx_group->coal.int_gl);
2658         }
2659
2660         tqp_vector->last_jiffies = jiffies;
2661         tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
2662 }
2663
2664 static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2665 {
2666         struct hns3_enet_ring *ring;
2667         int rx_pkt_total = 0;
2668
2669         struct hns3_enet_tqp_vector *tqp_vector =
2670                 container_of(napi, struct hns3_enet_tqp_vector, napi);
2671         bool clean_complete = true;
2672         int rx_budget;
2673
2674         /* Since the actual Tx work is minimal, we can give the Tx a larger
2675          * budget and be more aggressive about cleaning up the Tx descriptors.
2676          */
2677         hns3_for_each_ring(ring, tqp_vector->tx_group)
2678                 hns3_clean_tx_ring(ring);
2679
2680         /* make sure rx ring budget not smaller than 1 */
2681         rx_budget = max(budget / tqp_vector->num_tqps, 1);
2682
2683         hns3_for_each_ring(ring, tqp_vector->rx_group) {
2684                 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2685                                                     hns3_rx_skb);
2686
2687                 if (rx_cleaned >= rx_budget)
2688                         clean_complete = false;
2689
2690                 rx_pkt_total += rx_cleaned;
2691         }
2692
2693         tqp_vector->rx_group.total_packets += rx_pkt_total;
2694
2695         if (!clean_complete)
2696                 return budget;
2697
2698         napi_complete(napi);
2699         hns3_update_new_int_gl(tqp_vector);
2700         hns3_mask_vector_irq(tqp_vector, 1);
2701
2702         return rx_pkt_total;
2703 }
2704
2705 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2706                                       struct hnae3_ring_chain_node *head)
2707 {
2708         struct pci_dev *pdev = tqp_vector->handle->pdev;
2709         struct hnae3_ring_chain_node *cur_chain = head;
2710         struct hnae3_ring_chain_node *chain;
2711         struct hns3_enet_ring *tx_ring;
2712         struct hns3_enet_ring *rx_ring;
2713
2714         tx_ring = tqp_vector->tx_group.ring;
2715         if (tx_ring) {
2716                 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2717                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2718                               HNAE3_RING_TYPE_TX);
2719                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2720                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
2721
2722                 cur_chain->next = NULL;
2723
2724                 while (tx_ring->next) {
2725                         tx_ring = tx_ring->next;
2726
2727                         chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2728                                              GFP_KERNEL);
2729                         if (!chain)
2730                                 return -ENOMEM;
2731
2732                         cur_chain->next = chain;
2733                         chain->tqp_index = tx_ring->tqp->tqp_index;
2734                         hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2735                                       HNAE3_RING_TYPE_TX);
2736                         hnae3_set_field(chain->int_gl_idx,
2737                                         HNAE3_RING_GL_IDX_M,
2738                                         HNAE3_RING_GL_IDX_S,
2739                                         HNAE3_RING_GL_TX);
2740
2741                         cur_chain = chain;
2742                 }
2743         }
2744
2745         rx_ring = tqp_vector->rx_group.ring;
2746         if (!tx_ring && rx_ring) {
2747                 cur_chain->next = NULL;
2748                 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2749                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2750                               HNAE3_RING_TYPE_RX);
2751                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2752                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2753
2754                 rx_ring = rx_ring->next;
2755         }
2756
2757         while (rx_ring) {
2758                 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2759                 if (!chain)
2760                         return -ENOMEM;
2761
2762                 cur_chain->next = chain;
2763                 chain->tqp_index = rx_ring->tqp->tqp_index;
2764                 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2765                               HNAE3_RING_TYPE_RX);
2766                 hnae3_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2767                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2768
2769                 cur_chain = chain;
2770
2771                 rx_ring = rx_ring->next;
2772         }
2773
2774         return 0;
2775 }
2776
2777 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2778                                         struct hnae3_ring_chain_node *head)
2779 {
2780         struct pci_dev *pdev = tqp_vector->handle->pdev;
2781         struct hnae3_ring_chain_node *chain_tmp, *chain;
2782
2783         chain = head->next;
2784
2785         while (chain) {
2786                 chain_tmp = chain->next;
2787                 devm_kfree(&pdev->dev, chain);
2788                 chain = chain_tmp;
2789         }
2790 }
2791
2792 static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2793                                    struct hns3_enet_ring *ring)
2794 {
2795         ring->next = group->ring;
2796         group->ring = ring;
2797
2798         group->count++;
2799 }
2800
2801 static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv)
2802 {
2803         struct pci_dev *pdev = priv->ae_handle->pdev;
2804         struct hns3_enet_tqp_vector *tqp_vector;
2805         int num_vectors = priv->vector_num;
2806         int numa_node;
2807         int vector_i;
2808
2809         numa_node = dev_to_node(&pdev->dev);
2810
2811         for (vector_i = 0; vector_i < num_vectors; vector_i++) {
2812                 tqp_vector = &priv->tqp_vector[vector_i];
2813                 cpumask_set_cpu(cpumask_local_spread(vector_i, numa_node),
2814                                 &tqp_vector->affinity_mask);
2815         }
2816 }
2817
2818 static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2819 {
2820         struct hnae3_ring_chain_node vector_ring_chain;
2821         struct hnae3_handle *h = priv->ae_handle;
2822         struct hns3_enet_tqp_vector *tqp_vector;
2823         int ret = 0;
2824         u16 i;
2825
2826         hns3_nic_set_cpumask(priv);
2827
2828         for (i = 0; i < priv->vector_num; i++) {
2829                 tqp_vector = &priv->tqp_vector[i];
2830                 hns3_vector_gl_rl_init_hw(tqp_vector, priv);
2831                 tqp_vector->num_tqps = 0;
2832         }
2833
2834         for (i = 0; i < h->kinfo.num_tqps; i++) {
2835                 u16 vector_i = i % priv->vector_num;
2836                 u16 tqp_num = h->kinfo.num_tqps;
2837
2838                 tqp_vector = &priv->tqp_vector[vector_i];
2839
2840                 hns3_add_ring_to_group(&tqp_vector->tx_group,
2841                                        priv->ring_data[i].ring);
2842
2843                 hns3_add_ring_to_group(&tqp_vector->rx_group,
2844                                        priv->ring_data[i + tqp_num].ring);
2845
2846                 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2847                 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
2848                 tqp_vector->num_tqps++;
2849         }
2850
2851         for (i = 0; i < priv->vector_num; i++) {
2852                 tqp_vector = &priv->tqp_vector[i];
2853
2854                 tqp_vector->rx_group.total_bytes = 0;
2855                 tqp_vector->rx_group.total_packets = 0;
2856                 tqp_vector->tx_group.total_bytes = 0;
2857                 tqp_vector->tx_group.total_packets = 0;
2858                 tqp_vector->handle = h;
2859
2860                 ret = hns3_get_vector_ring_chain(tqp_vector,
2861                                                  &vector_ring_chain);
2862                 if (ret)
2863                         return ret;
2864
2865                 ret = h->ae_algo->ops->map_ring_to_vector(h,
2866                         tqp_vector->vector_irq, &vector_ring_chain);
2867
2868                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2869
2870                 if (ret)
2871                         return ret;
2872
2873                 netif_napi_add(priv->netdev, &tqp_vector->napi,
2874                                hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2875         }
2876
2877         return 0;
2878 }
2879
2880 static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
2881 {
2882         struct hnae3_handle *h = priv->ae_handle;
2883         struct hns3_enet_tqp_vector *tqp_vector;
2884         struct hnae3_vector_info *vector;
2885         struct pci_dev *pdev = h->pdev;
2886         u16 tqp_num = h->kinfo.num_tqps;
2887         u16 vector_num;
2888         int ret = 0;
2889         u16 i;
2890
2891         /* RSS size, cpu online and vector_num should be the same */
2892         /* Should consider 2p/4p later */
2893         vector_num = min_t(u16, num_online_cpus(), tqp_num);
2894         vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2895                               GFP_KERNEL);
2896         if (!vector)
2897                 return -ENOMEM;
2898
2899         vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2900
2901         priv->vector_num = vector_num;
2902         priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2903                 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2904                              GFP_KERNEL);
2905         if (!priv->tqp_vector) {
2906                 ret = -ENOMEM;
2907                 goto out;
2908         }
2909
2910         for (i = 0; i < priv->vector_num; i++) {
2911                 tqp_vector = &priv->tqp_vector[i];
2912                 tqp_vector->idx = i;
2913                 tqp_vector->mask_addr = vector[i].io_addr;
2914                 tqp_vector->vector_irq = vector[i].vector;
2915                 hns3_vector_gl_rl_init(tqp_vector, priv);
2916         }
2917
2918 out:
2919         devm_kfree(&pdev->dev, vector);
2920         return ret;
2921 }
2922
2923 static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
2924 {
2925         group->ring = NULL;
2926         group->count = 0;
2927 }
2928
2929 static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
2930 {
2931         struct hnae3_ring_chain_node vector_ring_chain;
2932         struct hnae3_handle *h = priv->ae_handle;
2933         struct hns3_enet_tqp_vector *tqp_vector;
2934         int i, ret;
2935
2936         for (i = 0; i < priv->vector_num; i++) {
2937                 tqp_vector = &priv->tqp_vector[i];
2938
2939                 ret = hns3_get_vector_ring_chain(tqp_vector,
2940                                                  &vector_ring_chain);
2941                 if (ret)
2942                         return ret;
2943
2944                 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
2945                         tqp_vector->vector_irq, &vector_ring_chain);
2946                 if (ret)
2947                         return ret;
2948
2949                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2950
2951                 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
2952                         (void)irq_set_affinity_hint(
2953                                 priv->tqp_vector[i].vector_irq,
2954                                                     NULL);
2955                         free_irq(priv->tqp_vector[i].vector_irq,
2956                                  &priv->tqp_vector[i]);
2957                 }
2958
2959                 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
2960                 hns3_clear_ring_group(&tqp_vector->rx_group);
2961                 hns3_clear_ring_group(&tqp_vector->tx_group);
2962                 netif_napi_del(&priv->tqp_vector[i].napi);
2963         }
2964
2965         return 0;
2966 }
2967
2968 static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
2969 {
2970         struct hnae3_handle *h = priv->ae_handle;
2971         struct pci_dev *pdev = h->pdev;
2972         int i, ret;
2973
2974         for (i = 0; i < priv->vector_num; i++) {
2975                 struct hns3_enet_tqp_vector *tqp_vector;
2976
2977                 tqp_vector = &priv->tqp_vector[i];
2978                 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
2979                 if (ret)
2980                         return ret;
2981         }
2982
2983         devm_kfree(&pdev->dev, priv->tqp_vector);
2984         return 0;
2985 }
2986
2987 static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
2988                              int ring_type)
2989 {
2990         struct hns3_nic_ring_data *ring_data = priv->ring_data;
2991         int queue_num = priv->ae_handle->kinfo.num_tqps;
2992         struct pci_dev *pdev = priv->ae_handle->pdev;
2993         struct hns3_enet_ring *ring;
2994
2995         ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
2996         if (!ring)
2997                 return -ENOMEM;
2998
2999         if (ring_type == HNAE3_RING_TYPE_TX) {
3000                 ring_data[q->tqp_index].ring = ring;
3001                 ring_data[q->tqp_index].queue_index = q->tqp_index;
3002                 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
3003         } else {
3004                 ring_data[q->tqp_index + queue_num].ring = ring;
3005                 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
3006                 ring->io_base = q->io_base;
3007         }
3008
3009         hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
3010
3011         ring->tqp = q;
3012         ring->desc = NULL;
3013         ring->desc_cb = NULL;
3014         ring->dev = priv->dev;
3015         ring->desc_dma_addr = 0;
3016         ring->buf_size = q->buf_size;
3017         ring->desc_num = q->desc_num;
3018         ring->next_to_use = 0;
3019         ring->next_to_clean = 0;
3020
3021         return 0;
3022 }
3023
3024 static int hns3_queue_to_ring(struct hnae3_queue *tqp,
3025                               struct hns3_nic_priv *priv)
3026 {
3027         int ret;
3028
3029         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
3030         if (ret)
3031                 return ret;
3032
3033         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
3034         if (ret)
3035                 return ret;
3036
3037         return 0;
3038 }
3039
3040 static int hns3_get_ring_config(struct hns3_nic_priv *priv)
3041 {
3042         struct hnae3_handle *h = priv->ae_handle;
3043         struct pci_dev *pdev = h->pdev;
3044         int i, ret;
3045
3046         priv->ring_data =  devm_kzalloc(&pdev->dev,
3047                                         array3_size(h->kinfo.num_tqps,
3048                                                     sizeof(*priv->ring_data),
3049                                                     2),
3050                                         GFP_KERNEL);
3051         if (!priv->ring_data)
3052                 return -ENOMEM;
3053
3054         for (i = 0; i < h->kinfo.num_tqps; i++) {
3055                 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
3056                 if (ret)
3057                         goto err;
3058         }
3059
3060         return 0;
3061 err:
3062         devm_kfree(&pdev->dev, priv->ring_data);
3063         return ret;
3064 }
3065
3066 static void hns3_put_ring_config(struct hns3_nic_priv *priv)
3067 {
3068         struct hnae3_handle *h = priv->ae_handle;
3069         int i;
3070
3071         for (i = 0; i < h->kinfo.num_tqps; i++) {
3072                 devm_kfree(priv->dev, priv->ring_data[i].ring);
3073                 devm_kfree(priv->dev,
3074                            priv->ring_data[i + h->kinfo.num_tqps].ring);
3075         }
3076         devm_kfree(priv->dev, priv->ring_data);
3077 }
3078
3079 static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
3080 {
3081         int ret;
3082
3083         if (ring->desc_num <= 0 || ring->buf_size <= 0)
3084                 return -EINVAL;
3085
3086         ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
3087                                 GFP_KERNEL);
3088         if (!ring->desc_cb) {
3089                 ret = -ENOMEM;
3090                 goto out;
3091         }
3092
3093         ret = hns3_alloc_desc(ring);
3094         if (ret)
3095                 goto out_with_desc_cb;
3096
3097         if (!HNAE3_IS_TX_RING(ring)) {
3098                 ret = hns3_alloc_ring_buffers(ring);
3099                 if (ret)
3100                         goto out_with_desc;
3101         }
3102
3103         return 0;
3104
3105 out_with_desc:
3106         hns3_free_desc(ring);
3107 out_with_desc_cb:
3108         kfree(ring->desc_cb);
3109         ring->desc_cb = NULL;
3110 out:
3111         return ret;
3112 }
3113
3114 static void hns3_fini_ring(struct hns3_enet_ring *ring)
3115 {
3116         hns3_free_desc(ring);
3117         kfree(ring->desc_cb);
3118         ring->desc_cb = NULL;
3119         ring->next_to_clean = 0;
3120         ring->next_to_use = 0;
3121 }
3122
3123 static int hns3_buf_size2type(u32 buf_size)
3124 {
3125         int bd_size_type;
3126
3127         switch (buf_size) {
3128         case 512:
3129                 bd_size_type = HNS3_BD_SIZE_512_TYPE;
3130                 break;
3131         case 1024:
3132                 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
3133                 break;
3134         case 2048:
3135                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3136                 break;
3137         case 4096:
3138                 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
3139                 break;
3140         default:
3141                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3142         }
3143
3144         return bd_size_type;
3145 }
3146
3147 static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
3148 {
3149         dma_addr_t dma = ring->desc_dma_addr;
3150         struct hnae3_queue *q = ring->tqp;
3151
3152         if (!HNAE3_IS_TX_RING(ring)) {
3153                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
3154                                (u32)dma);
3155                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
3156                                (u32)((dma >> 31) >> 1));
3157
3158                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
3159                                hns3_buf_size2type(ring->buf_size));
3160                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
3161                                ring->desc_num / 8 - 1);
3162
3163         } else {
3164                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
3165                                (u32)dma);
3166                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
3167                                (u32)((dma >> 31) >> 1));
3168
3169                 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
3170                                ring->desc_num / 8 - 1);
3171         }
3172 }
3173
3174 static void hns3_init_tx_ring_tc(struct hns3_nic_priv *priv)
3175 {
3176         struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
3177         int i;
3178
3179         for (i = 0; i < HNAE3_MAX_TC; i++) {
3180                 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3181                 int j;
3182
3183                 if (!tc_info->enable)
3184                         continue;
3185
3186                 for (j = 0; j < tc_info->tqp_count; j++) {
3187                         struct hnae3_queue *q;
3188
3189                         q = priv->ring_data[tc_info->tqp_offset + j].ring->tqp;
3190                         hns3_write_dev(q, HNS3_RING_TX_RING_TC_REG,
3191                                        tc_info->tc);
3192                 }
3193         }
3194 }
3195
3196 int hns3_init_all_ring(struct hns3_nic_priv *priv)
3197 {
3198         struct hnae3_handle *h = priv->ae_handle;
3199         int ring_num = h->kinfo.num_tqps * 2;
3200         int i, j;
3201         int ret;
3202
3203         for (i = 0; i < ring_num; i++) {
3204                 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
3205                 if (ret) {
3206                         dev_err(priv->dev,
3207                                 "Alloc ring memory fail! ret=%d\n", ret);
3208                         goto out_when_alloc_ring_memory;
3209                 }
3210
3211                 u64_stats_init(&priv->ring_data[i].ring->syncp);
3212         }
3213
3214         return 0;
3215
3216 out_when_alloc_ring_memory:
3217         for (j = i - 1; j >= 0; j--)
3218                 hns3_fini_ring(priv->ring_data[j].ring);
3219
3220         return -ENOMEM;
3221 }
3222
3223 int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
3224 {
3225         struct hnae3_handle *h = priv->ae_handle;
3226         int i;
3227
3228         for (i = 0; i < h->kinfo.num_tqps; i++) {
3229                 if (h->ae_algo->ops->reset_queue)
3230                         h->ae_algo->ops->reset_queue(h, i);
3231
3232                 hns3_fini_ring(priv->ring_data[i].ring);
3233                 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
3234         }
3235         return 0;
3236 }
3237
3238 /* Set mac addr if it is configured. or leave it to the AE driver */
3239 static void hns3_init_mac_addr(struct net_device *netdev, bool init)
3240 {
3241         struct hns3_nic_priv *priv = netdev_priv(netdev);
3242         struct hnae3_handle *h = priv->ae_handle;
3243         u8 mac_addr_temp[ETH_ALEN];
3244
3245         if (h->ae_algo->ops->get_mac_addr && init) {
3246                 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
3247                 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
3248         }
3249
3250         /* Check if the MAC address is valid, if not get a random one */
3251         if (!is_valid_ether_addr(netdev->dev_addr)) {
3252                 eth_hw_addr_random(netdev);
3253                 dev_warn(priv->dev, "using random MAC address %pM\n",
3254                          netdev->dev_addr);
3255         }
3256
3257         if (h->ae_algo->ops->set_mac_addr)
3258                 h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true);
3259
3260 }
3261
3262 static int hns3_restore_fd_rules(struct net_device *netdev)
3263 {
3264         struct hnae3_handle *h = hns3_get_handle(netdev);
3265         int ret = 0;
3266
3267         if (h->ae_algo->ops->restore_fd_rules)
3268                 ret = h->ae_algo->ops->restore_fd_rules(h);
3269
3270         return ret;
3271 }
3272
3273 static void hns3_del_all_fd_rules(struct net_device *netdev, bool clear_list)
3274 {
3275         struct hnae3_handle *h = hns3_get_handle(netdev);
3276
3277         if (h->ae_algo->ops->del_all_fd_entries)
3278                 h->ae_algo->ops->del_all_fd_entries(h, clear_list);
3279 }
3280
3281 static void hns3_nic_set_priv_ops(struct net_device *netdev)
3282 {
3283         struct hns3_nic_priv *priv = netdev_priv(netdev);
3284
3285         priv->ops.fill_desc = hns3_fill_desc;
3286         if ((netdev->features & NETIF_F_TSO) ||
3287             (netdev->features & NETIF_F_TSO6))
3288                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
3289         else
3290                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
3291 }
3292
3293 static int hns3_client_init(struct hnae3_handle *handle)
3294 {
3295         struct pci_dev *pdev = handle->pdev;
3296         u16 alloc_tqps, max_rss_size;
3297         struct hns3_nic_priv *priv;
3298         struct net_device *netdev;
3299         int ret;
3300
3301         handle->ae_algo->ops->get_tqps_and_rss_info(handle, &alloc_tqps,
3302                                                     &max_rss_size);
3303         netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv), alloc_tqps);
3304         if (!netdev)
3305                 return -ENOMEM;
3306
3307         priv = netdev_priv(netdev);
3308         priv->dev = &pdev->dev;
3309         priv->netdev = netdev;
3310         priv->ae_handle = handle;
3311         priv->ae_handle->last_reset_time = jiffies;
3312         priv->tx_timeout_count = 0;
3313
3314         handle->kinfo.netdev = netdev;
3315         handle->priv = (void *)priv;
3316
3317         hns3_init_mac_addr(netdev, true);
3318
3319         hns3_set_default_feature(netdev);
3320
3321         netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
3322         netdev->priv_flags |= IFF_UNICAST_FLT;
3323         netdev->netdev_ops = &hns3_nic_netdev_ops;
3324         SET_NETDEV_DEV(netdev, &pdev->dev);
3325         hns3_ethtool_set_ops(netdev);
3326         hns3_nic_set_priv_ops(netdev);
3327
3328         /* Carrier off reporting is important to ethtool even BEFORE open */
3329         netif_carrier_off(netdev);
3330
3331         if (handle->flags & HNAE3_SUPPORT_VF)
3332                 handle->reset_level = HNAE3_VF_RESET;
3333         else
3334                 handle->reset_level = HNAE3_FUNC_RESET;
3335
3336         ret = hns3_get_ring_config(priv);
3337         if (ret) {
3338                 ret = -ENOMEM;
3339                 goto out_get_ring_cfg;
3340         }
3341
3342         ret = hns3_nic_alloc_vector_data(priv);
3343         if (ret) {
3344                 ret = -ENOMEM;
3345                 goto out_alloc_vector_data;
3346         }
3347
3348         ret = hns3_nic_init_vector_data(priv);
3349         if (ret) {
3350                 ret = -ENOMEM;
3351                 goto out_init_vector_data;
3352         }
3353
3354         ret = hns3_init_all_ring(priv);
3355         if (ret) {
3356                 ret = -ENOMEM;
3357                 goto out_init_ring_data;
3358         }
3359
3360         ret = register_netdev(netdev);
3361         if (ret) {
3362                 dev_err(priv->dev, "probe register netdev fail!\n");
3363                 goto out_reg_netdev_fail;
3364         }
3365
3366         hns3_dcbnl_setup(handle);
3367
3368         /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */
3369         netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
3370
3371         return ret;
3372
3373 out_reg_netdev_fail:
3374 out_init_ring_data:
3375         (void)hns3_nic_uninit_vector_data(priv);
3376 out_init_vector_data:
3377         hns3_nic_dealloc_vector_data(priv);
3378 out_alloc_vector_data:
3379         priv->ring_data = NULL;
3380 out_get_ring_cfg:
3381         priv->ae_handle = NULL;
3382         free_netdev(netdev);
3383         return ret;
3384 }
3385
3386 static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
3387 {
3388         struct net_device *netdev = handle->kinfo.netdev;
3389         struct hns3_nic_priv *priv = netdev_priv(netdev);
3390         int ret;
3391
3392         hns3_remove_hw_addr(netdev);
3393
3394         if (netdev->reg_state != NETREG_UNINITIALIZED)
3395                 unregister_netdev(netdev);
3396
3397         hns3_del_all_fd_rules(netdev, true);
3398
3399         hns3_force_clear_all_rx_ring(handle);
3400
3401         ret = hns3_nic_uninit_vector_data(priv);
3402         if (ret)
3403                 netdev_err(netdev, "uninit vector error\n");
3404
3405         ret = hns3_nic_dealloc_vector_data(priv);
3406         if (ret)
3407                 netdev_err(netdev, "dealloc vector error\n");
3408
3409         ret = hns3_uninit_all_ring(priv);
3410         if (ret)
3411                 netdev_err(netdev, "uninit ring error\n");
3412
3413         hns3_put_ring_config(priv);
3414
3415         priv->ring_data = NULL;
3416
3417         free_netdev(netdev);
3418 }
3419
3420 static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
3421 {
3422         struct net_device *netdev = handle->kinfo.netdev;
3423
3424         if (!netdev)
3425                 return;
3426
3427         if (linkup) {
3428                 netif_carrier_on(netdev);
3429                 netif_tx_wake_all_queues(netdev);
3430                 netdev_info(netdev, "link up\n");
3431         } else {
3432                 netif_carrier_off(netdev);
3433                 netif_tx_stop_all_queues(netdev);
3434                 netdev_info(netdev, "link down\n");
3435         }
3436 }
3437
3438 static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
3439 {
3440         struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3441         struct net_device *ndev = kinfo->netdev;
3442         bool if_running;
3443         int ret;
3444
3445         if (tc > HNAE3_MAX_TC)
3446                 return -EINVAL;
3447
3448         if (!ndev)
3449                 return -ENODEV;
3450
3451         if_running = netif_running(ndev);
3452
3453         if (if_running) {
3454                 (void)hns3_nic_net_stop(ndev);
3455                 msleep(100);
3456         }
3457
3458         ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ?
3459                 kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP;
3460         if (ret)
3461                 goto err_out;
3462
3463         ret = hns3_nic_set_real_num_queue(ndev);
3464
3465 err_out:
3466         if (if_running)
3467                 (void)hns3_nic_net_open(ndev);
3468
3469         return ret;
3470 }
3471
3472 static void hns3_recover_hw_addr(struct net_device *ndev)
3473 {
3474         struct netdev_hw_addr_list *list;
3475         struct netdev_hw_addr *ha, *tmp;
3476
3477         /* go through and sync uc_addr entries to the device */
3478         list = &ndev->uc;
3479         list_for_each_entry_safe(ha, tmp, &list->list, list)
3480                 hns3_nic_uc_sync(ndev, ha->addr);
3481
3482         /* go through and sync mc_addr entries to the device */
3483         list = &ndev->mc;
3484         list_for_each_entry_safe(ha, tmp, &list->list, list)
3485                 hns3_nic_mc_sync(ndev, ha->addr);
3486 }
3487
3488 static void hns3_remove_hw_addr(struct net_device *netdev)
3489 {
3490         struct netdev_hw_addr_list *list;
3491         struct netdev_hw_addr *ha, *tmp;
3492
3493         hns3_nic_uc_unsync(netdev, netdev->dev_addr);
3494
3495         /* go through and unsync uc_addr entries to the device */
3496         list = &netdev->uc;
3497         list_for_each_entry_safe(ha, tmp, &list->list, list)
3498                 hns3_nic_uc_unsync(netdev, ha->addr);
3499
3500         /* go through and unsync mc_addr entries to the device */
3501         list = &netdev->mc;
3502         list_for_each_entry_safe(ha, tmp, &list->list, list)
3503                 if (ha->refcount > 1)
3504                         hns3_nic_mc_unsync(netdev, ha->addr);
3505 }
3506
3507 static void hns3_clear_tx_ring(struct hns3_enet_ring *ring)
3508 {
3509         while (ring->next_to_clean != ring->next_to_use) {
3510                 ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0;
3511                 hns3_free_buffer_detach(ring, ring->next_to_clean);
3512                 ring_ptr_move_fw(ring, next_to_clean);
3513         }
3514 }
3515
3516 static int hns3_clear_rx_ring(struct hns3_enet_ring *ring)
3517 {
3518         struct hns3_desc_cb res_cbs;
3519         int ret;
3520
3521         while (ring->next_to_use != ring->next_to_clean) {
3522                 /* When a buffer is not reused, it's memory has been
3523                  * freed in hns3_handle_rx_bd or will be freed by
3524                  * stack, so we need to replace the buffer here.
3525                  */
3526                 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3527                         ret = hns3_reserve_buffer_map(ring, &res_cbs);
3528                         if (ret) {
3529                                 u64_stats_update_begin(&ring->syncp);
3530                                 ring->stats.sw_err_cnt++;
3531                                 u64_stats_update_end(&ring->syncp);
3532                                 /* if alloc new buffer fail, exit directly
3533                                  * and reclear in up flow.
3534                                  */
3535                                 netdev_warn(ring->tqp->handle->kinfo.netdev,
3536                                             "reserve buffer map failed, ret = %d\n",
3537                                             ret);
3538                                 return ret;
3539                         }
3540                         hns3_replace_buffer(ring, ring->next_to_use,
3541                                             &res_cbs);
3542                 }
3543                 ring_ptr_move_fw(ring, next_to_use);
3544         }
3545
3546         return 0;
3547 }
3548
3549 static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring)
3550 {
3551         while (ring->next_to_use != ring->next_to_clean) {
3552                 /* When a buffer is not reused, it's memory has been
3553                  * freed in hns3_handle_rx_bd or will be freed by
3554                  * stack, so only need to unmap the buffer here.
3555                  */
3556                 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3557                         hns3_unmap_buffer(ring,
3558                                           &ring->desc_cb[ring->next_to_use]);
3559                         ring->desc_cb[ring->next_to_use].dma = 0;
3560                 }
3561
3562                 ring_ptr_move_fw(ring, next_to_use);
3563         }
3564 }
3565
3566 static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h)
3567 {
3568         struct net_device *ndev = h->kinfo.netdev;
3569         struct hns3_nic_priv *priv = netdev_priv(ndev);
3570         struct hns3_enet_ring *ring;
3571         u32 i;
3572
3573         for (i = 0; i < h->kinfo.num_tqps; i++) {
3574                 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3575                 hns3_force_clear_rx_ring(ring);
3576         }
3577 }
3578
3579 static void hns3_clear_all_ring(struct hnae3_handle *h)
3580 {
3581         struct net_device *ndev = h->kinfo.netdev;
3582         struct hns3_nic_priv *priv = netdev_priv(ndev);
3583         u32 i;
3584
3585         for (i = 0; i < h->kinfo.num_tqps; i++) {
3586                 struct netdev_queue *dev_queue;
3587                 struct hns3_enet_ring *ring;
3588
3589                 ring = priv->ring_data[i].ring;
3590                 hns3_clear_tx_ring(ring);
3591                 dev_queue = netdev_get_tx_queue(ndev,
3592                                                 priv->ring_data[i].queue_index);
3593                 netdev_tx_reset_queue(dev_queue);
3594
3595                 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3596                 /* Continue to clear other rings even if clearing some
3597                  * rings failed.
3598                  */
3599                 hns3_clear_rx_ring(ring);
3600         }
3601 }
3602
3603 int hns3_nic_reset_all_ring(struct hnae3_handle *h)
3604 {
3605         struct net_device *ndev = h->kinfo.netdev;
3606         struct hns3_nic_priv *priv = netdev_priv(ndev);
3607         struct hns3_enet_ring *rx_ring;
3608         int i, j;
3609         int ret;
3610
3611         for (i = 0; i < h->kinfo.num_tqps; i++) {
3612                 h->ae_algo->ops->reset_queue(h, i);
3613                 hns3_init_ring_hw(priv->ring_data[i].ring);
3614
3615                 /* We need to clear tx ring here because self test will
3616                  * use the ring and will not run down before up
3617                  */
3618                 hns3_clear_tx_ring(priv->ring_data[i].ring);
3619                 priv->ring_data[i].ring->next_to_clean = 0;
3620                 priv->ring_data[i].ring->next_to_use = 0;
3621
3622                 rx_ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3623                 hns3_init_ring_hw(rx_ring);
3624                 ret = hns3_clear_rx_ring(rx_ring);
3625                 if (ret)
3626                         return ret;
3627
3628                 /* We can not know the hardware head and tail when this
3629                  * function is called in reset flow, so we reuse all desc.
3630                  */
3631                 for (j = 0; j < rx_ring->desc_num; j++)
3632                         hns3_reuse_buffer(rx_ring, j);
3633
3634                 rx_ring->next_to_clean = 0;
3635                 rx_ring->next_to_use = 0;
3636         }
3637
3638         hns3_init_tx_ring_tc(priv);
3639
3640         return 0;
3641 }
3642
3643 static void hns3_store_coal(struct hns3_nic_priv *priv)
3644 {
3645         /* ethtool only support setting and querying one coal
3646          * configuation for now, so save the vector 0' coal
3647          * configuation here in order to restore it.
3648          */
3649         memcpy(&priv->tx_coal, &priv->tqp_vector[0].tx_group.coal,
3650                sizeof(struct hns3_enet_coalesce));
3651         memcpy(&priv->rx_coal, &priv->tqp_vector[0].rx_group.coal,
3652                sizeof(struct hns3_enet_coalesce));
3653 }
3654
3655 static void hns3_restore_coal(struct hns3_nic_priv *priv)
3656 {
3657         u16 vector_num = priv->vector_num;
3658         int i;
3659
3660         for (i = 0; i < vector_num; i++) {
3661                 memcpy(&priv->tqp_vector[i].tx_group.coal, &priv->tx_coal,
3662                        sizeof(struct hns3_enet_coalesce));
3663                 memcpy(&priv->tqp_vector[i].rx_group.coal, &priv->rx_coal,
3664                        sizeof(struct hns3_enet_coalesce));
3665         }
3666 }
3667
3668 static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
3669 {
3670         struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3671         struct net_device *ndev = kinfo->netdev;
3672
3673         if (!netif_running(ndev))
3674                 return 0;
3675
3676         return hns3_nic_net_stop(ndev);
3677 }
3678
3679 static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
3680 {
3681         struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3682         int ret = 0;
3683
3684         if (netif_running(kinfo->netdev)) {
3685                 ret = hns3_nic_net_up(kinfo->netdev);
3686                 if (ret) {
3687                         netdev_err(kinfo->netdev,
3688                                    "hns net up fail, ret=%d!\n", ret);
3689                         return ret;
3690                 }
3691                 handle->last_reset_time = jiffies;
3692         }
3693
3694         return ret;
3695 }
3696
3697 static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
3698 {
3699         struct net_device *netdev = handle->kinfo.netdev;
3700         struct hns3_nic_priv *priv = netdev_priv(netdev);
3701         bool vlan_filter_enable;
3702         int ret;
3703
3704         hns3_init_mac_addr(netdev, false);
3705         hns3_recover_hw_addr(netdev);
3706         hns3_update_promisc_mode(netdev, handle->netdev_flags);
3707         vlan_filter_enable = netdev->flags & IFF_PROMISC ? false : true;
3708         hns3_enable_vlan_filter(netdev, vlan_filter_enable);
3709
3710
3711         /* Hardware table is only clear when pf resets */
3712         if (!(handle->flags & HNAE3_SUPPORT_VF))
3713                 hns3_restore_vlan(netdev);
3714
3715         hns3_restore_fd_rules(netdev);
3716
3717         /* Carrier off reporting is important to ethtool even BEFORE open */
3718         netif_carrier_off(netdev);
3719
3720         hns3_restore_coal(priv);
3721
3722         ret = hns3_nic_init_vector_data(priv);
3723         if (ret)
3724                 return ret;
3725
3726         ret = hns3_init_all_ring(priv);
3727         if (ret) {
3728                 hns3_nic_uninit_vector_data(priv);
3729                 priv->ring_data = NULL;
3730         }
3731
3732         return ret;
3733 }
3734
3735 static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
3736 {
3737         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev);
3738         struct net_device *netdev = handle->kinfo.netdev;
3739         struct hns3_nic_priv *priv = netdev_priv(netdev);
3740         int ret;
3741
3742         hns3_force_clear_all_rx_ring(handle);
3743
3744         ret = hns3_nic_uninit_vector_data(priv);
3745         if (ret) {
3746                 netdev_err(netdev, "uninit vector error\n");
3747                 return ret;
3748         }
3749
3750         hns3_store_coal(priv);
3751
3752         ret = hns3_uninit_all_ring(priv);
3753         if (ret)
3754                 netdev_err(netdev, "uninit ring error\n");
3755
3756         /* it is cumbersome for hardware to pick-and-choose entries for deletion
3757          * from table space. Hence, for function reset software intervention is
3758          * required to delete the entries
3759          */
3760         if (hns3_dev_ongoing_func_reset(ae_dev)) {
3761                 hns3_remove_hw_addr(netdev);
3762                 hns3_del_all_fd_rules(netdev, false);
3763         }
3764
3765         return ret;
3766 }
3767
3768 static int hns3_reset_notify(struct hnae3_handle *handle,
3769                              enum hnae3_reset_notify_type type)
3770 {
3771         int ret = 0;
3772
3773         switch (type) {
3774         case HNAE3_UP_CLIENT:
3775                 ret = hns3_reset_notify_up_enet(handle);
3776                 break;
3777         case HNAE3_DOWN_CLIENT:
3778                 ret = hns3_reset_notify_down_enet(handle);
3779                 break;
3780         case HNAE3_INIT_CLIENT:
3781                 ret = hns3_reset_notify_init_enet(handle);
3782                 break;
3783         case HNAE3_UNINIT_CLIENT:
3784                 ret = hns3_reset_notify_uninit_enet(handle);
3785                 break;
3786         default:
3787                 break;
3788         }
3789
3790         return ret;
3791 }
3792
3793 static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num)
3794 {
3795         struct hns3_nic_priv *priv = netdev_priv(netdev);
3796         struct hnae3_handle *h = hns3_get_handle(netdev);
3797         int ret;
3798
3799         ret = h->ae_algo->ops->set_channels(h, new_tqp_num);
3800         if (ret)
3801                 return ret;
3802
3803         ret = hns3_get_ring_config(priv);
3804         if (ret)
3805                 return ret;
3806
3807         ret = hns3_nic_alloc_vector_data(priv);
3808         if (ret)
3809                 goto err_alloc_vector;
3810
3811         hns3_restore_coal(priv);
3812
3813         ret = hns3_nic_init_vector_data(priv);
3814         if (ret)
3815                 goto err_uninit_vector;
3816
3817         ret = hns3_init_all_ring(priv);
3818         if (ret)
3819                 goto err_put_ring;
3820
3821         return 0;
3822
3823 err_put_ring:
3824         hns3_put_ring_config(priv);
3825 err_uninit_vector:
3826         hns3_nic_uninit_vector_data(priv);
3827 err_alloc_vector:
3828         hns3_nic_dealloc_vector_data(priv);
3829         return ret;
3830 }
3831
3832 static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num)
3833 {
3834         return (new_tqp_num / num_tc) * num_tc;
3835 }
3836
3837 int hns3_set_channels(struct net_device *netdev,
3838                       struct ethtool_channels *ch)
3839 {
3840         struct hns3_nic_priv *priv = netdev_priv(netdev);
3841         struct hnae3_handle *h = hns3_get_handle(netdev);
3842         struct hnae3_knic_private_info *kinfo = &h->kinfo;
3843         bool if_running = netif_running(netdev);
3844         u32 new_tqp_num = ch->combined_count;
3845         u16 org_tqp_num;
3846         int ret;
3847
3848         if (ch->rx_count || ch->tx_count)
3849                 return -EINVAL;
3850
3851         if (new_tqp_num > hns3_get_max_available_channels(h) ||
3852             new_tqp_num < kinfo->num_tc) {
3853                 dev_err(&netdev->dev,
3854                         "Change tqps fail, the tqp range is from %d to %d",
3855                         kinfo->num_tc,
3856                         hns3_get_max_available_channels(h));
3857                 return -EINVAL;
3858         }
3859
3860         new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num);
3861         if (kinfo->num_tqps == new_tqp_num)
3862                 return 0;
3863
3864         if (if_running)
3865                 hns3_nic_net_stop(netdev);
3866
3867         ret = hns3_nic_uninit_vector_data(priv);
3868         if (ret) {
3869                 dev_err(&netdev->dev,
3870                         "Unbind vector with tqp fail, nothing is changed");
3871                 goto open_netdev;
3872         }
3873
3874         hns3_store_coal(priv);
3875
3876         hns3_nic_dealloc_vector_data(priv);
3877
3878         hns3_uninit_all_ring(priv);
3879         hns3_put_ring_config(priv);
3880
3881         org_tqp_num = h->kinfo.num_tqps;
3882         ret = hns3_modify_tqp_num(netdev, new_tqp_num);
3883         if (ret) {
3884                 ret = hns3_modify_tqp_num(netdev, org_tqp_num);
3885                 if (ret) {
3886                         /* If revert to old tqp failed, fatal error occurred */
3887                         dev_err(&netdev->dev,
3888                                 "Revert to old tqp num fail, ret=%d", ret);
3889                         return ret;
3890                 }
3891                 dev_info(&netdev->dev,
3892                          "Change tqp num fail, Revert to old tqp num");
3893         }
3894
3895 open_netdev:
3896         if (if_running)
3897                 hns3_nic_net_open(netdev);
3898
3899         return ret;
3900 }
3901
3902 static const struct hnae3_client_ops client_ops = {
3903         .init_instance = hns3_client_init,
3904         .uninit_instance = hns3_client_uninit,
3905         .link_status_change = hns3_link_status_change,
3906         .setup_tc = hns3_client_setup_tc,
3907         .reset_notify = hns3_reset_notify,
3908 };
3909
3910 /* hns3_init_module - Driver registration routine
3911  * hns3_init_module is the first routine called when the driver is
3912  * loaded. All it does is register with the PCI subsystem.
3913  */
3914 static int __init hns3_init_module(void)
3915 {
3916         int ret;
3917
3918         pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
3919         pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
3920
3921         client.type = HNAE3_CLIENT_KNIC;
3922         snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
3923                  hns3_driver_name);
3924
3925         client.ops = &client_ops;
3926
3927         INIT_LIST_HEAD(&client.node);
3928
3929         ret = hnae3_register_client(&client);
3930         if (ret)
3931                 return ret;
3932
3933         ret = pci_register_driver(&hns3_driver);
3934         if (ret)
3935                 hnae3_unregister_client(&client);
3936
3937         return ret;
3938 }
3939 module_init(hns3_init_module);
3940
3941 /* hns3_exit_module - Driver exit cleanup routine
3942  * hns3_exit_module is called just before the driver is removed
3943  * from memory.
3944  */
3945 static void __exit hns3_exit_module(void)
3946 {
3947         pci_unregister_driver(&hns3_driver);
3948         hnae3_unregister_client(&client);
3949 }
3950 module_exit(hns3_exit_module);
3951
3952 MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
3953 MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
3954 MODULE_LICENSE("GPL");
3955 MODULE_ALIAS("pci:hns-nic");
3956 MODULE_VERSION(HNS3_MOD_VERSION);