drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.c

   1 /*
   2  * Copyright (c) 2016~2017 Hisilicon Limited.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  */
   9
  10 #include <linux/dma-mapping.h>
  11 #include <linux/etherdevice.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/if_vlan.h>
  14 #include <linux/ip.h>
  15 #include <linux/ipv6.h>
  16 #include <linux/module.h>
  17 #include <linux/pci.h>
  18 #include <linux/skbuff.h>
  19 #include <linux/sctp.h>
  20 #include <linux/vermagic.h>
  21 #include <net/gre.h>
  22 #include <net/vxlan.h>
  23
  24 #include "hnae3.h"
  25 #include "hns3_enet.h"
  26
  27 const char hns3_driver_name[] = "hns3";
  28 const char hns3_driver_version[] = VERMAGIC_STRING;
  29 static const char hns3_driver_string[] =
  30                         "Hisilicon Ethernet Network Driver for Hip08 Family";
  31 static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
  32 static struct hnae3_client client;
  33
  34 /* hns3_pci_tbl - PCI Device ID Table
  35  *
  36  * Last entry must be all 0s
  37  *
  38  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  39  *   Class, Class Mask, private data (not used) }
  40  */
  41 static const struct pci_device_id hns3_pci_tbl[] = {
  42         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
  43         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
  44         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
  45         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
  46         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
  47         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
  48         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
  49         /* required last entry */
  50         {0, }
  51 };
  52 MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
  53
  54 static irqreturn_t hns3_irq_handle(int irq, void *dev)
  55 {
  56         struct hns3_enet_tqp_vector *tqp_vector = dev;
  57
  58         napi_schedule(&tqp_vector->napi);
  59
  60         return IRQ_HANDLED;
  61 }
  62
  63 static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
  64 {
  65         struct hns3_enet_tqp_vector *tqp_vectors;
  66         unsigned int i;
  67
  68         for (i = 0; i < priv->vector_num; i++) {
  69                 tqp_vectors = &priv->tqp_vector[i];
  70
  71                 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
  72                         continue;
  73
  74                 /* release the irq resource */
  75                 free_irq(tqp_vectors->vector_irq, tqp_vectors);
  76                 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
  77         }
  78 }
  79
  80 static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
  81 {
  82         struct hns3_enet_tqp_vector *tqp_vectors;
  83         int txrx_int_idx = 0;
  84         int rx_int_idx = 0;
  85         int tx_int_idx = 0;
  86         unsigned int i;
  87         int ret;
  88
  89         for (i = 0; i < priv->vector_num; i++) {
  90                 tqp_vectors = &priv->tqp_vector[i];
  91
  92                 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
  93                         continue;
  94
  95                 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
  96                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
  97                                  "%s-%s-%d", priv->netdev->name, "TxRx",
  98                                  txrx_int_idx++);
  99                         txrx_int_idx++;
 100                 } else if (tqp_vectors->rx_group.ring) {
 101                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
 102                                  "%s-%s-%d", priv->netdev->name, "Rx",
 103                                  rx_int_idx++);
 104                 } else if (tqp_vectors->tx_group.ring) {
 105                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
 106                                  "%s-%s-%d", priv->netdev->name, "Tx",
 107                                  tx_int_idx++);
 108                 } else {
 109                         /* Skip this unused q_vector */
 110                         continue;
 111                 }
 112
 113                 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
 114
 115                 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
 116                                   tqp_vectors->name,
 117                                        tqp_vectors);
 118                 if (ret) {
 119                         netdev_err(priv->netdev, "request irq(%d) fail\n",
 120                                    tqp_vectors->vector_irq);
 121                         return ret;
 122                 }
 123
 124                 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
 125         }
 126
 127         return 0;
 128 }
 129
 130 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
 131                                  u32 mask_en)
 132 {
 133         writel(mask_en, tqp_vector->mask_addr);
 134 }
 135
 136 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
 137 {
 138         napi_enable(&tqp_vector->napi);
 139
 140         /* enable vector */
 141         hns3_mask_vector_irq(tqp_vector, 1);
 142 }
 143
 144 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
 145 {
 146         /* disable vector */
 147         hns3_mask_vector_irq(tqp_vector, 0);
 148
 149         disable_irq(tqp_vector->vector_irq);
 150         napi_disable(&tqp_vector->napi);
 151 }
 152
 153 static void hns3_set_vector_coalesc_gl(struct hns3_enet_tqp_vector *tqp_vector,
 154                                        u32 gl_value)
 155 {
 156         /* this defines the configuration for GL (Interrupt Gap Limiter)
 157          * GL defines inter interrupt gap.
 158          * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
 159          */
 160         writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
 161         writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
 162         writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL2_OFFSET);
 163 }
 164
 165 static void hns3_set_vector_coalesc_rl(struct hns3_enet_tqp_vector *tqp_vector,
 166                                        u32 rl_value)
 167 {
 168         /* this defines the configuration for RL (Interrupt Rate Limiter).
 169          * Rl defines rate of interrupts i.e. number of interrupts-per-second
 170          * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
 171          */
 172         writel(rl_value, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
 173 }
 174
 175 static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector)
 176 {
 177         /* initialize the configuration for interrupt coalescing.
 178          * 1. GL (Interrupt Gap Limiter)
 179          * 2. RL (Interrupt Rate Limiter)
 180          */
 181
 182         /* Default :enable interrupt coalesce */
 183         tqp_vector->rx_group.int_gl = HNS3_INT_GL_50K;
 184         tqp_vector->tx_group.int_gl = HNS3_INT_GL_50K;
 185         hns3_set_vector_coalesc_gl(tqp_vector, HNS3_INT_GL_50K);
 186         /* for now we are disabling Interrupt RL - we
 187          * will re-enable later
 188          */
 189         hns3_set_vector_coalesc_rl(tqp_vector, 0);
 190         tqp_vector->rx_group.flow_level = HNS3_FLOW_LOW;
 191         tqp_vector->tx_group.flow_level = HNS3_FLOW_LOW;
 192 }
 193
 194 static int hns3_nic_net_up(struct net_device *netdev)
 195 {
 196         struct hns3_nic_priv *priv = netdev_priv(netdev);
 197         struct hnae3_handle *h = priv->ae_handle;
 198         int i, j;
 199         int ret;
 200
 201         /* get irq resource for all vectors */
 202         ret = hns3_nic_init_irq(priv);
 203         if (ret) {
 204                 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
 205                 return ret;
 206         }
 207
 208         /* enable the vectors */
 209         for (i = 0; i < priv->vector_num; i++)
 210                 hns3_vector_enable(&priv->tqp_vector[i]);
 211
 212         /* start the ae_dev */
 213         ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
 214         if (ret)
 215                 goto out_start_err;
 216
 217         return 0;
 218
 219 out_start_err:
 220         for (j = i - 1; j >= 0; j--)
 221                 hns3_vector_disable(&priv->tqp_vector[j]);
 222
 223         hns3_nic_uninit_irq(priv);
 224
 225         return ret;
 226 }
 227
 228 static int hns3_nic_net_open(struct net_device *netdev)
 229 {
 230         struct hns3_nic_priv *priv = netdev_priv(netdev);
 231         struct hnae3_handle *h = priv->ae_handle;
 232         int ret;
 233
 234         netif_carrier_off(netdev);
 235
 236         ret = netif_set_real_num_tx_queues(netdev, h->kinfo.num_tqps);
 237         if (ret) {
 238                 netdev_err(netdev,
 239                            "netif_set_real_num_tx_queues fail, ret=%d!\n",
 240                            ret);
 241                 return ret;
 242         }
 243
 244         ret = netif_set_real_num_rx_queues(netdev, h->kinfo.num_tqps);
 245         if (ret) {
 246                 netdev_err(netdev,
 247                            "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
 248                 return ret;
 249         }
 250
 251         ret = hns3_nic_net_up(netdev);
 252         if (ret) {
 253                 netdev_err(netdev,
 254                            "hns net up fail, ret=%d!\n", ret);
 255                 return ret;
 256         }
 257
 258         return 0;
 259 }
 260
 261 static void hns3_nic_net_down(struct net_device *netdev)
 262 {
 263         struct hns3_nic_priv *priv = netdev_priv(netdev);
 264         const struct hnae3_ae_ops *ops;
 265         int i;
 266
 267         /* stop ae_dev */
 268         ops = priv->ae_handle->ae_algo->ops;
 269         if (ops->stop)
 270                 ops->stop(priv->ae_handle);
 271
 272         /* disable vectors */
 273         for (i = 0; i < priv->vector_num; i++)
 274                 hns3_vector_disable(&priv->tqp_vector[i]);
 275
 276         /* free irq resources */
 277         hns3_nic_uninit_irq(priv);
 278 }
 279
 280 static int hns3_nic_net_stop(struct net_device *netdev)
 281 {
 282         netif_tx_stop_all_queues(netdev);
 283         netif_carrier_off(netdev);
 284
 285         hns3_nic_net_down(netdev);
 286
 287         return 0;
 288 }
 289
 290 void hns3_set_multicast_list(struct net_device *netdev)
 291 {
 292         struct hns3_nic_priv *priv = netdev_priv(netdev);
 293         struct hnae3_handle *h = priv->ae_handle;
 294         struct netdev_hw_addr *ha = NULL;
 295
 296         if (h->ae_algo->ops->set_mc_addr) {
 297                 netdev_for_each_mc_addr(ha, netdev)
 298                         if (h->ae_algo->ops->set_mc_addr(h, ha->addr))
 299                                 netdev_err(netdev, "set multicast fail\n");
 300         }
 301 }
 302
 303 static int hns3_nic_uc_sync(struct net_device *netdev,
 304                             const unsigned char *addr)
 305 {
 306         struct hns3_nic_priv *priv = netdev_priv(netdev);
 307         struct hnae3_handle *h = priv->ae_handle;
 308
 309         if (h->ae_algo->ops->add_uc_addr)
 310                 return h->ae_algo->ops->add_uc_addr(h, addr);
 311
 312         return 0;
 313 }
 314
 315 static int hns3_nic_uc_unsync(struct net_device *netdev,
 316                               const unsigned char *addr)
 317 {
 318         struct hns3_nic_priv *priv = netdev_priv(netdev);
 319         struct hnae3_handle *h = priv->ae_handle;
 320
 321         if (h->ae_algo->ops->rm_uc_addr)
 322                 return h->ae_algo->ops->rm_uc_addr(h, addr);
 323
 324         return 0;
 325 }
 326
 327 static int hns3_nic_mc_sync(struct net_device *netdev,
 328                             const unsigned char *addr)
 329 {
 330         struct hns3_nic_priv *priv = netdev_priv(netdev);
 331         struct hnae3_handle *h = priv->ae_handle;
 332
 333         if (h->ae_algo->ops->add_uc_addr)
 334                 return h->ae_algo->ops->add_mc_addr(h, addr);
 335
 336         return 0;
 337 }
 338
 339 static int hns3_nic_mc_unsync(struct net_device *netdev,
 340                               const unsigned char *addr)
 341 {
 342         struct hns3_nic_priv *priv = netdev_priv(netdev);
 343         struct hnae3_handle *h = priv->ae_handle;
 344
 345         if (h->ae_algo->ops->rm_uc_addr)
 346                 return h->ae_algo->ops->rm_mc_addr(h, addr);
 347
 348         return 0;
 349 }
 350
 351 void hns3_nic_set_rx_mode(struct net_device *netdev)
 352 {
 353         struct hns3_nic_priv *priv = netdev_priv(netdev);
 354         struct hnae3_handle *h = priv->ae_handle;
 355
 356         if (h->ae_algo->ops->set_promisc_mode) {
 357                 if (netdev->flags & IFF_PROMISC)
 358                         h->ae_algo->ops->set_promisc_mode(h, 1);
 359                 else
 360                         h->ae_algo->ops->set_promisc_mode(h, 0);
 361         }
 362         if (__dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync))
 363                 netdev_err(netdev, "sync uc address fail\n");
 364         if (netdev->flags & IFF_MULTICAST)
 365                 if (__dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync))
 366                         netdev_err(netdev, "sync mc address fail\n");
 367 }
 368
 369 static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
 370                         u16 *mss, u32 *type_cs_vlan_tso)
 371 {
 372         u32 l4_offset, hdr_len;
 373         union l3_hdr_info l3;
 374         union l4_hdr_info l4;
 375         u32 l4_paylen;
 376         int ret;
 377
 378         if (!skb_is_gso(skb))
 379                 return 0;
 380
 381         ret = skb_cow_head(skb, 0);
 382         if (ret)
 383                 return ret;
 384
 385         l3.hdr = skb_network_header(skb);
 386         l4.hdr = skb_transport_header(skb);
 387
 388         /* Software should clear the IPv4's checksum field when tso is
 389          * needed.
 390          */
 391         if (l3.v4->version == 4)
 392                 l3.v4->check = 0;
 393
 394         /* tunnel packet.*/
 395         if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
 396                                          SKB_GSO_GRE_CSUM |
 397                                          SKB_GSO_UDP_TUNNEL |
 398                                          SKB_GSO_UDP_TUNNEL_CSUM)) {
 399                 if ((!(skb_shinfo(skb)->gso_type &
 400                     SKB_GSO_PARTIAL)) &&
 401                     (skb_shinfo(skb)->gso_type &
 402                     SKB_GSO_UDP_TUNNEL_CSUM)) {
 403                         /* Software should clear the udp's checksum
 404                          * field when tso is needed.
 405                          */
 406                         l4.udp->check = 0;
 407                 }
 408                 /* reset l3&l4 pointers from outer to inner headers */
 409                 l3.hdr = skb_inner_network_header(skb);
 410                 l4.hdr = skb_inner_transport_header(skb);
 411
 412                 /* Software should clear the IPv4's checksum field when
 413                  * tso is needed.
 414                  */
 415                 if (l3.v4->version == 4)
 416                         l3.v4->check = 0;
 417         }
 418
 419         /* normal or tunnel packet*/
 420         l4_offset = l4.hdr - skb->data;
 421         hdr_len = (l4.tcp->doff * 4) + l4_offset;
 422
 423         /* remove payload length from inner pseudo checksum when tso*/
 424         l4_paylen = skb->len - l4_offset;
 425         csum_replace_by_diff(&l4.tcp->check,
 426                              (__force __wsum)htonl(l4_paylen));
 427
 428         /* find the txbd field values */
 429         *paylen = skb->len - hdr_len;
 430         hnae_set_bit(*type_cs_vlan_tso,
 431                      HNS3_TXD_TSO_B, 1);
 432
 433         /* get MSS for TSO */
 434         *mss = skb_shinfo(skb)->gso_size;
 435
 436         return 0;
 437 }
 438
 439 static void hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
 440                                  u8 *il4_proto)
 441 {
 442         union {
 443                 struct iphdr *v4;
 444                 struct ipv6hdr *v6;
 445                 unsigned char *hdr;
 446         } l3;
 447         unsigned char *l4_hdr;
 448         unsigned char *exthdr;
 449         u8 l4_proto_tmp;
 450         __be16 frag_off;
 451
 452         /* find outer header point */
 453         l3.hdr = skb_network_header(skb);
 454         l4_hdr = skb_inner_transport_header(skb);
 455
 456         if (skb->protocol == htons(ETH_P_IPV6)) {
 457                 exthdr = l3.hdr + sizeof(*l3.v6);
 458                 l4_proto_tmp = l3.v6->nexthdr;
 459                 if (l4_hdr != exthdr)
 460                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 461                                          &l4_proto_tmp, &frag_off);
 462         } else if (skb->protocol == htons(ETH_P_IP)) {
 463                 l4_proto_tmp = l3.v4->protocol;
 464         }
 465
 466         *ol4_proto = l4_proto_tmp;
 467
 468         /* tunnel packet */
 469         if (!skb->encapsulation) {
 470                 *il4_proto = 0;
 471                 return;
 472         }
 473
 474         /* find inner header point */
 475         l3.hdr = skb_inner_network_header(skb);
 476         l4_hdr = skb_inner_transport_header(skb);
 477
 478         if (l3.v6->version == 6) {
 479                 exthdr = l3.hdr + sizeof(*l3.v6);
 480                 l4_proto_tmp = l3.v6->nexthdr;
 481                 if (l4_hdr != exthdr)
 482                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 483                                          &l4_proto_tmp, &frag_off);
 484         } else if (l3.v4->version == 4) {
 485                 l4_proto_tmp = l3.v4->protocol;
 486         }
 487
 488         *il4_proto = l4_proto_tmp;
 489 }
 490
 491 static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
 492                                 u8 il4_proto, u32 *type_cs_vlan_tso,
 493                                 u32 *ol_type_vlan_len_msec)
 494 {
 495         union {
 496                 struct iphdr *v4;
 497                 struct ipv6hdr *v6;
 498                 unsigned char *hdr;
 499         } l3;
 500         union {
 501                 struct tcphdr *tcp;
 502                 struct udphdr *udp;
 503                 struct gre_base_hdr *gre;
 504                 unsigned char *hdr;
 505         } l4;
 506         unsigned char *l2_hdr;
 507         u8 l4_proto = ol4_proto;
 508         u32 ol2_len;
 509         u32 ol3_len;
 510         u32 ol4_len;
 511         u32 l2_len;
 512         u32 l3_len;
 513
 514         l3.hdr = skb_network_header(skb);
 515         l4.hdr = skb_transport_header(skb);
 516
 517         /* compute L2 header size for normal packet, defined in 2 Bytes */
 518         l2_len = l3.hdr - skb->data;
 519         hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
 520                        HNS3_TXD_L2LEN_S, l2_len >> 1);
 521
 522         /* tunnel packet*/
 523         if (skb->encapsulation) {
 524                 /* compute OL2 header size, defined in 2 Bytes */
 525                 ol2_len = l2_len;
 526                 hnae_set_field(*ol_type_vlan_len_msec,
 527                                HNS3_TXD_L2LEN_M,
 528                                HNS3_TXD_L2LEN_S, ol2_len >> 1);
 529
 530                 /* compute OL3 header size, defined in 4 Bytes */
 531                 ol3_len = l4.hdr - l3.hdr;
 532                 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
 533                                HNS3_TXD_L3LEN_S, ol3_len >> 2);
 534
 535                 /* MAC in UDP, MAC in GRE (0x6558)*/
 536                 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
 537                         /* switch MAC header ptr from outer to inner header.*/
 538                         l2_hdr = skb_inner_mac_header(skb);
 539
 540                         /* compute OL4 header size, defined in 4 Bytes. */
 541                         ol4_len = l2_hdr - l4.hdr;
 542                         hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_M,
 543                                        HNS3_TXD_L4LEN_S, ol4_len >> 2);
 544
 545                         /* switch IP header ptr from outer to inner header */
 546                         l3.hdr = skb_inner_network_header(skb);
 547
 548                         /* compute inner l2 header size, defined in 2 Bytes. */
 549                         l2_len = l3.hdr - l2_hdr;
 550                         hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
 551                                        HNS3_TXD_L2LEN_S, l2_len >> 1);
 552                 } else {
 553                         /* skb packet types not supported by hardware,
 554                          * txbd len fild doesn't be filled.
 555                          */
 556                         return;
 557                 }
 558
 559                 /* switch L4 header pointer from outer to inner */
 560                 l4.hdr = skb_inner_transport_header(skb);
 561
 562                 l4_proto = il4_proto;
 563         }
 564
 565         /* compute inner(/normal) L3 header size, defined in 4 Bytes */
 566         l3_len = l4.hdr - l3.hdr;
 567         hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
 568                        HNS3_TXD_L3LEN_S, l3_len >> 2);
 569
 570         /* compute inner(/normal) L4 header size, defined in 4 Bytes */
 571         switch (l4_proto) {
 572         case IPPROTO_TCP:
 573                 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 574                                HNS3_TXD_L4LEN_S, l4.tcp->doff);
 575                 break;
 576         case IPPROTO_SCTP:
 577                 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 578                                HNS3_TXD_L4LEN_S, (sizeof(struct sctphdr) >> 2));
 579                 break;
 580         case IPPROTO_UDP:
 581                 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 582                                HNS3_TXD_L4LEN_S, (sizeof(struct udphdr) >> 2));
 583                 break;
 584         default:
 585                 /* skb packet types not supported by hardware,
 586                  * txbd len fild doesn't be filled.
 587                  */
 588                 return;
 589         }
 590 }
 591
 592 static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
 593                                    u8 il4_proto, u32 *type_cs_vlan_tso,
 594                                    u32 *ol_type_vlan_len_msec)
 595 {
 596         union {
 597                 struct iphdr *v4;
 598                 struct ipv6hdr *v6;
 599                 unsigned char *hdr;
 600         } l3;
 601         u32 l4_proto = ol4_proto;
 602
 603         l3.hdr = skb_network_header(skb);
 604
 605         /* define OL3 type and tunnel type(OL4).*/
 606         if (skb->encapsulation) {
 607                 /* define outer network header type.*/
 608                 if (skb->protocol == htons(ETH_P_IP)) {
 609                         if (skb_is_gso(skb))
 610                                 hnae_set_field(*ol_type_vlan_len_msec,
 611                                                HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
 612                                                HNS3_OL3T_IPV4_CSUM);
 613                         else
 614                                 hnae_set_field(*ol_type_vlan_len_msec,
 615                                                HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
 616                                                HNS3_OL3T_IPV4_NO_CSUM);
 617
 618                 } else if (skb->protocol == htons(ETH_P_IPV6)) {
 619                         hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
 620                                        HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
 621                 }
 622
 623                 /* define tunnel type(OL4).*/
 624                 switch (l4_proto) {
 625                 case IPPROTO_UDP:
 626                         hnae_set_field(*ol_type_vlan_len_msec,
 627                                        HNS3_TXD_TUNTYPE_M,
 628                                        HNS3_TXD_TUNTYPE_S,
 629                                        HNS3_TUN_MAC_IN_UDP);
 630                         break;
 631                 case IPPROTO_GRE:
 632                         hnae_set_field(*ol_type_vlan_len_msec,
 633                                        HNS3_TXD_TUNTYPE_M,
 634                                        HNS3_TXD_TUNTYPE_S,
 635                                        HNS3_TUN_NVGRE);
 636                         break;
 637                 default:
 638                         /* drop the skb tunnel packet if hardware don't support,
 639                          * because hardware can't calculate csum when TSO.
 640                          */
 641                         if (skb_is_gso(skb))
 642                                 return -EDOM;
 643
 644                         /* the stack computes the IP header already,
 645                          * driver calculate l4 checksum when not TSO.
 646                          */
 647                         skb_checksum_help(skb);
 648                         return 0;
 649                 }
 650
 651                 l3.hdr = skb_inner_network_header(skb);
 652                 l4_proto = il4_proto;
 653         }
 654
 655         if (l3.v4->version == 4) {
 656                 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
 657                                HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
 658
 659                 /* the stack computes the IP header already, the only time we
 660                  * need the hardware to recompute it is in the case of TSO.
 661                  */
 662                 if (skb_is_gso(skb))
 663                         hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
 664
 665                 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 666         } else if (l3.v6->version == 6) {
 667                 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
 668                                HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
 669                 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 670         }
 671
 672         switch (l4_proto) {
 673         case IPPROTO_TCP:
 674                 hnae_set_field(*type_cs_vlan_tso,
 675                                HNS3_TXD_L4T_M,
 676                                HNS3_TXD_L4T_S,
 677                                HNS3_L4T_TCP);
 678                 break;
 679         case IPPROTO_UDP:
 680                 hnae_set_field(*type_cs_vlan_tso,
 681                                HNS3_TXD_L4T_M,
 682                                HNS3_TXD_L4T_S,
 683                                HNS3_L4T_UDP);
 684                 break;
 685         case IPPROTO_SCTP:
 686                 hnae_set_field(*type_cs_vlan_tso,
 687                                HNS3_TXD_L4T_M,
 688                                HNS3_TXD_L4T_S,
 689                                HNS3_L4T_SCTP);
 690                 break;
 691         default:
 692                 /* drop the skb tunnel packet if hardware don't support,
 693                  * because hardware can't calculate csum when TSO.
 694                  */
 695                 if (skb_is_gso(skb))
 696                         return -EDOM;
 697
 698                 /* the stack computes the IP header already,
 699                  * driver calculate l4 checksum when not TSO.
 700                  */
 701                 skb_checksum_help(skb);
 702                 return 0;
 703         }
 704
 705         return 0;
 706 }
 707
 708 static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
 709 {
 710         /* Config bd buffer end */
 711         hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
 712                        HNS3_TXD_BDTYPE_M, 0);
 713         hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
 714         hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
 715         hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 1);
 716 }
 717
 718 static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
 719                           int size, dma_addr_t dma, int frag_end,
 720                           enum hns_desc_type type)
 721 {
 722         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
 723         struct hns3_desc *desc = &ring->desc[ring->next_to_use];
 724         u32 ol_type_vlan_len_msec = 0;
 725         u16 bdtp_fe_sc_vld_ra_ri = 0;
 726         u32 type_cs_vlan_tso = 0;
 727         struct sk_buff *skb;
 728         u32 paylen = 0;
 729         u16 mss = 0;
 730         __be16 protocol;
 731         u8 ol4_proto;
 732         u8 il4_proto;
 733         int ret;
 734
 735         /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
 736         desc_cb->priv = priv;
 737         desc_cb->length = size;
 738         desc_cb->dma = dma;
 739         desc_cb->type = type;
 740
 741         /* now, fill the descriptor */
 742         desc->addr = cpu_to_le64(dma);
 743         desc->tx.send_size = cpu_to_le16((u16)size);
 744         hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri, frag_end);
 745         desc->tx.bdtp_fe_sc_vld_ra_ri = cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
 746
 747         if (type == DESC_TYPE_SKB) {
 748                 skb = (struct sk_buff *)priv;
 749                 paylen = cpu_to_le16(skb->len);
 750
 751                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
 752                         skb_reset_mac_len(skb);
 753                         protocol = skb->protocol;
 754
 755                         /* vlan packet*/
 756                         if (protocol == htons(ETH_P_8021Q)) {
 757                                 protocol = vlan_get_protocol(skb);
 758                                 skb->protocol = protocol;
 759                         }
 760                         hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
 761                         hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
 762                                             &type_cs_vlan_tso,
 763                                             &ol_type_vlan_len_msec);
 764                         ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
 765                                                       &type_cs_vlan_tso,
 766                                                       &ol_type_vlan_len_msec);
 767                         if (ret)
 768                                 return ret;
 769
 770                         ret = hns3_set_tso(skb, &paylen, &mss,
 771                                            &type_cs_vlan_tso);
 772                         if (ret)
 773                                 return ret;
 774                 }
 775
 776                 /* Set txbd */
 777                 desc->tx.ol_type_vlan_len_msec =
 778                         cpu_to_le32(ol_type_vlan_len_msec);
 779                 desc->tx.type_cs_vlan_tso_len =
 780                         cpu_to_le32(type_cs_vlan_tso);
 781                 desc->tx.paylen = cpu_to_le16(paylen);
 782                 desc->tx.mss = cpu_to_le16(mss);
 783         }
 784
 785         /* move ring pointer to next.*/
 786         ring_ptr_move_fw(ring, next_to_use);
 787
 788         return 0;
 789 }
 790
 791 static int hns3_fill_desc_tso(struct hns3_enet_ring *ring, void *priv,
 792                               int size, dma_addr_t dma, int frag_end,
 793                               enum hns_desc_type type)
 794 {
 795         unsigned int frag_buf_num;
 796         unsigned int k;
 797         int sizeoflast;
 798         int ret;
 799
 800         frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
 801         sizeoflast = size % HNS3_MAX_BD_SIZE;
 802         sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
 803
 804         /* When the frag size is bigger than hardware, split this frag */
 805         for (k = 0; k < frag_buf_num; k++) {
 806                 ret = hns3_fill_desc(ring, priv,
 807                                      (k == frag_buf_num - 1) ?
 808                                 sizeoflast : HNS3_MAX_BD_SIZE,
 809                                 dma + HNS3_MAX_BD_SIZE * k,
 810                                 frag_end && (k == frag_buf_num - 1) ? 1 : 0,
 811                                 (type == DESC_TYPE_SKB && !k) ?
 812                                         DESC_TYPE_SKB : DESC_TYPE_PAGE);
 813                 if (ret)
 814                         return ret;
 815         }
 816
 817         return 0;
 818 }
 819
 820 static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
 821                                    struct hns3_enet_ring *ring)
 822 {
 823         struct sk_buff *skb = *out_skb;
 824         struct skb_frag_struct *frag;
 825         int bdnum_for_frag;
 826         int frag_num;
 827         int buf_num;
 828         int size;
 829         int i;
 830
 831         size = skb_headlen(skb);
 832         buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
 833
 834         frag_num = skb_shinfo(skb)->nr_frags;
 835         for (i = 0; i < frag_num; i++) {
 836                 frag = &skb_shinfo(skb)->frags[i];
 837                 size = skb_frag_size(frag);
 838                 bdnum_for_frag =
 839                         (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
 840                 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
 841                         return -ENOMEM;
 842
 843                 buf_num += bdnum_for_frag;
 844         }
 845
 846         if (buf_num > ring_space(ring))
 847                 return -EBUSY;
 848
 849         *bnum = buf_num;
 850         return 0;
 851 }
 852
 853 static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
 854                                   struct hns3_enet_ring *ring)
 855 {
 856         struct sk_buff *skb = *out_skb;
 857         int buf_num;
 858
 859         /* No. of segments (plus a header) */
 860         buf_num = skb_shinfo(skb)->nr_frags + 1;
 861
 862         if (buf_num > ring_space(ring))
 863                 return -EBUSY;
 864
 865         *bnum = buf_num;
 866
 867         return 0;
 868 }
 869
 870 static void hns_nic_dma_unmap(struct hns3_enet_ring *ring, int next_to_use_orig)
 871 {
 872         struct device *dev = ring_to_dev(ring);
 873         unsigned int i;
 874
 875         for (i = 0; i < ring->desc_num; i++) {
 876                 /* check if this is where we started */
 877                 if (ring->next_to_use == next_to_use_orig)
 878                         break;
 879
 880                 /* unmap the descriptor dma address */
 881                 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
 882                         dma_unmap_single(dev,
 883                                          ring->desc_cb[ring->next_to_use].dma,
 884                                         ring->desc_cb[ring->next_to_use].length,
 885                                         DMA_TO_DEVICE);
 886                 else
 887                         dma_unmap_page(dev,
 888                                        ring->desc_cb[ring->next_to_use].dma,
 889                                        ring->desc_cb[ring->next_to_use].length,
 890                                        DMA_TO_DEVICE);
 891
 892                 /* rollback one */
 893                 ring_ptr_move_bw(ring, next_to_use);
 894         }
 895 }
 896
 897 static netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb,
 898                                      struct net_device *netdev)
 899 {
 900         struct hns3_nic_priv *priv = netdev_priv(netdev);
 901         struct hns3_nic_ring_data *ring_data =
 902                 &tx_ring_data(priv, skb->queue_mapping);
 903         struct hns3_enet_ring *ring = ring_data->ring;
 904         struct device *dev = priv->dev;
 905         struct netdev_queue *dev_queue;
 906         struct skb_frag_struct *frag;
 907         int next_to_use_head;
 908         int next_to_use_frag;
 909         dma_addr_t dma;
 910         int buf_num;
 911         int seg_num;
 912         int size;
 913         int ret;
 914         int i;
 915
 916         /* Prefetch the data used later */
 917         prefetch(skb->data);
 918
 919         switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
 920         case -EBUSY:
 921                 u64_stats_update_begin(&ring->syncp);
 922                 ring->stats.tx_busy++;
 923                 u64_stats_update_end(&ring->syncp);
 924
 925                 goto out_net_tx_busy;
 926         case -ENOMEM:
 927                 u64_stats_update_begin(&ring->syncp);
 928                 ring->stats.sw_err_cnt++;
 929                 u64_stats_update_end(&ring->syncp);
 930                 netdev_err(netdev, "no memory to xmit!\n");
 931
 932                 goto out_err_tx_ok;
 933         default:
 934                 break;
 935         }
 936
 937         /* No. of segments (plus a header) */
 938         seg_num = skb_shinfo(skb)->nr_frags + 1;
 939         /* Fill the first part */
 940         size = skb_headlen(skb);
 941
 942         next_to_use_head = ring->next_to_use;
 943
 944         dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
 945         if (dma_mapping_error(dev, dma)) {
 946                 netdev_err(netdev, "TX head DMA map failed\n");
 947                 ring->stats.sw_err_cnt++;
 948                 goto out_err_tx_ok;
 949         }
 950
 951         ret = priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0,
 952                            DESC_TYPE_SKB);
 953         if (ret)
 954                 goto head_dma_map_err;
 955
 956         next_to_use_frag = ring->next_to_use;
 957         /* Fill the fragments */
 958         for (i = 1; i < seg_num; i++) {
 959                 frag = &skb_shinfo(skb)->frags[i - 1];
 960                 size = skb_frag_size(frag);
 961                 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
 962                 if (dma_mapping_error(dev, dma)) {
 963                         netdev_err(netdev, "TX frag(%d) DMA map failed\n", i);
 964                         ring->stats.sw_err_cnt++;
 965                         goto frag_dma_map_err;
 966                 }
 967                 ret = priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma,
 968                                     seg_num - 1 == i ? 1 : 0,
 969                                     DESC_TYPE_PAGE);
 970
 971                 if (ret)
 972                         goto frag_dma_map_err;
 973         }
 974
 975         /* Complete translate all packets */
 976         dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
 977         netdev_tx_sent_queue(dev_queue, skb->len);
 978
 979         wmb(); /* Commit all data before submit */
 980
 981         hnae_queue_xmit(ring->tqp, buf_num);
 982
 983         return NETDEV_TX_OK;
 984
 985 frag_dma_map_err:
 986         hns_nic_dma_unmap(ring, next_to_use_frag);
 987
 988 head_dma_map_err:
 989         hns_nic_dma_unmap(ring, next_to_use_head);
 990
 991 out_err_tx_ok:
 992         dev_kfree_skb_any(skb);
 993         return NETDEV_TX_OK;
 994
 995 out_net_tx_busy:
 996         netif_stop_subqueue(netdev, ring_data->queue_index);
 997         smp_mb(); /* Commit all data before submit */
 998
 999         return NETDEV_TX_BUSY;
1000 }
1001
1002 static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1003 {
1004         struct hns3_nic_priv *priv = netdev_priv(netdev);
1005         struct hnae3_handle *h = priv->ae_handle;
1006         struct sockaddr *mac_addr = p;
1007         int ret;
1008
1009         if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1010                 return -EADDRNOTAVAIL;
1011
1012         ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data);
1013         if (ret) {
1014                 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1015                 return ret;
1016         }
1017
1018         ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1019
1020         return 0;
1021 }
1022
1023 static int hns3_nic_set_features(struct net_device *netdev,
1024                                  netdev_features_t features)
1025 {
1026         struct hns3_nic_priv *priv = netdev_priv(netdev);
1027
1028         if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
1029                 priv->ops.fill_desc = hns3_fill_desc_tso;
1030                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1031         } else {
1032                 priv->ops.fill_desc = hns3_fill_desc;
1033                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1034         }
1035
1036         netdev->features = features;
1037         return 0;
1038 }
1039
1040 static void
1041 hns3_nic_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
1042 {
1043         struct hns3_nic_priv *priv = netdev_priv(netdev);
1044         int queue_num = priv->ae_handle->kinfo.num_tqps;
1045         struct hns3_enet_ring *ring;
1046         unsigned int start;
1047         unsigned int idx;
1048         u64 tx_bytes = 0;
1049         u64 rx_bytes = 0;
1050         u64 tx_pkts = 0;
1051         u64 rx_pkts = 0;
1052
1053         for (idx = 0; idx < queue_num; idx++) {
1054                 /* fetch the tx stats */
1055                 ring = priv->ring_data[idx].ring;
1056                 do {
1057                         tx_bytes += ring->stats.tx_bytes;
1058                         tx_pkts += ring->stats.tx_pkts;
1059                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1060
1061                 /* fetch the rx stats */
1062                 ring = priv->ring_data[idx + queue_num].ring;
1063                 do {
1064                         rx_bytes += ring->stats.rx_bytes;
1065                         rx_pkts += ring->stats.rx_pkts;
1066                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1067         }
1068
1069         stats->tx_bytes = tx_bytes;
1070         stats->tx_packets = tx_pkts;
1071         stats->rx_bytes = rx_bytes;
1072         stats->rx_packets = rx_pkts;
1073
1074         stats->rx_errors = netdev->stats.rx_errors;
1075         stats->multicast = netdev->stats.multicast;
1076         stats->rx_length_errors = netdev->stats.rx_length_errors;
1077         stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1078         stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1079
1080         stats->tx_errors = netdev->stats.tx_errors;
1081         stats->rx_dropped = netdev->stats.rx_dropped;
1082         stats->tx_dropped = netdev->stats.tx_dropped;
1083         stats->collisions = netdev->stats.collisions;
1084         stats->rx_over_errors = netdev->stats.rx_over_errors;
1085         stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1086         stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1087         stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1088         stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1089         stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1090         stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1091         stats->tx_window_errors = netdev->stats.tx_window_errors;
1092         stats->rx_compressed = netdev->stats.rx_compressed;
1093         stats->tx_compressed = netdev->stats.tx_compressed;
1094 }
1095
1096 static void hns3_add_tunnel_port(struct net_device *netdev, u16 port,
1097                                  enum hns3_udp_tnl_type type)
1098 {
1099         struct hns3_nic_priv *priv = netdev_priv(netdev);
1100         struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1101         struct hnae3_handle *h = priv->ae_handle;
1102
1103         if (udp_tnl->used && udp_tnl->dst_port == port) {
1104                 udp_tnl->used++;
1105                 return;
1106         }
1107
1108         if (udp_tnl->used) {
1109                 netdev_warn(netdev,
1110                             "UDP tunnel [%d], port [%d] offload\n", type, port);
1111                 return;
1112         }
1113
1114         udp_tnl->dst_port = port;
1115         udp_tnl->used = 1;
1116         /* TBD send command to hardware to add port */
1117         if (h->ae_algo->ops->add_tunnel_udp)
1118                 h->ae_algo->ops->add_tunnel_udp(h, port);
1119 }
1120
1121 static void hns3_del_tunnel_port(struct net_device *netdev, u16 port,
1122                                  enum hns3_udp_tnl_type type)
1123 {
1124         struct hns3_nic_priv *priv = netdev_priv(netdev);
1125         struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1126         struct hnae3_handle *h = priv->ae_handle;
1127
1128         if (!udp_tnl->used || udp_tnl->dst_port != port) {
1129                 netdev_warn(netdev,
1130                             "Invalid UDP tunnel port %d\n", port);
1131                 return;
1132         }
1133
1134         udp_tnl->used--;
1135         if (udp_tnl->used)
1136                 return;
1137
1138         udp_tnl->dst_port = 0;
1139         /* TBD send command to hardware to del port  */
1140         if (h->ae_algo->ops->del_tunnel_udp)
1141                 h->ae_algo->ops->del_tunnel_udp(h, port);
1142 }
1143
1144 /* hns3_nic_udp_tunnel_add - Get notifiacetion about UDP tunnel ports
1145  * @netdev: This physical ports's netdev
1146  * @ti: Tunnel information
1147  */
1148 static void hns3_nic_udp_tunnel_add(struct net_device *netdev,
1149                                     struct udp_tunnel_info *ti)
1150 {
1151         u16 port_n = ntohs(ti->port);
1152
1153         switch (ti->type) {
1154         case UDP_TUNNEL_TYPE_VXLAN:
1155                 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1156                 break;
1157         case UDP_TUNNEL_TYPE_GENEVE:
1158                 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1159                 break;
1160         default:
1161                 netdev_err(netdev, "unsupported tunnel type %d\n", ti->type);
1162                 break;
1163         }
1164 }
1165
1166 static void hns3_nic_udp_tunnel_del(struct net_device *netdev,
1167                                     struct udp_tunnel_info *ti)
1168 {
1169         u16 port_n = ntohs(ti->port);
1170
1171         switch (ti->type) {
1172         case UDP_TUNNEL_TYPE_VXLAN:
1173                 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1174                 break;
1175         case UDP_TUNNEL_TYPE_GENEVE:
1176                 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1177                 break;
1178         default:
1179                 break;
1180         }
1181 }
1182
1183 static int hns3_setup_tc(struct net_device *netdev, u8 tc)
1184 {
1185         struct hns3_nic_priv *priv = netdev_priv(netdev);
1186         struct hnae3_handle *h = priv->ae_handle;
1187         struct hnae3_knic_private_info *kinfo = &h->kinfo;
1188         unsigned int i;
1189         int ret;
1190
1191         if (tc > HNAE3_MAX_TC)
1192                 return -EINVAL;
1193
1194         if (kinfo->num_tc == tc)
1195                 return 0;
1196
1197         if (!netdev)
1198                 return -EINVAL;
1199
1200         if (!tc) {
1201                 netdev_reset_tc(netdev);
1202                 return 0;
1203         }
1204
1205         /* Set num_tc for netdev */
1206         ret = netdev_set_num_tc(netdev, tc);
1207         if (ret)
1208                 return ret;
1209
1210         /* Set per TC queues for the VSI */
1211         for (i = 0; i < HNAE3_MAX_TC; i++) {
1212                 if (kinfo->tc_info[i].enable)
1213                         netdev_set_tc_queue(netdev,
1214                                             kinfo->tc_info[i].tc,
1215                                             kinfo->tc_info[i].tqp_count,
1216                                             kinfo->tc_info[i].tqp_offset);
1217         }
1218
1219         return 0;
1220 }
1221
1222 static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
1223                              void *type_data)
1224 {
1225         struct tc_mqprio_qopt *mqprio = type_data;
1226
1227         if (type != TC_SETUP_MQPRIO)
1228                 return -EOPNOTSUPP;
1229
1230         return hns3_setup_tc(dev, mqprio->num_tc);
1231 }
1232
1233 static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1234                                 __be16 proto, u16 vid)
1235 {
1236         struct hns3_nic_priv *priv = netdev_priv(netdev);
1237         struct hnae3_handle *h = priv->ae_handle;
1238         int ret = -EIO;
1239
1240         if (h->ae_algo->ops->set_vlan_filter)
1241                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1242
1243         return ret;
1244 }
1245
1246 static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1247                                  __be16 proto, u16 vid)
1248 {
1249         struct hns3_nic_priv *priv = netdev_priv(netdev);
1250         struct hnae3_handle *h = priv->ae_handle;
1251         int ret = -EIO;
1252
1253         if (h->ae_algo->ops->set_vlan_filter)
1254                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1255
1256         return ret;
1257 }
1258
1259 static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1260                                 u8 qos, __be16 vlan_proto)
1261 {
1262         struct hns3_nic_priv *priv = netdev_priv(netdev);
1263         struct hnae3_handle *h = priv->ae_handle;
1264         int ret = -EIO;
1265
1266         if (h->ae_algo->ops->set_vf_vlan_filter)
1267                 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1268                                                    qos, vlan_proto);
1269
1270         return ret;
1271 }
1272
1273 static const struct net_device_ops hns3_nic_netdev_ops = {
1274         .ndo_open               = hns3_nic_net_open,
1275         .ndo_stop               = hns3_nic_net_stop,
1276         .ndo_start_xmit         = hns3_nic_net_xmit,
1277         .ndo_set_mac_address    = hns3_nic_net_set_mac_address,
1278         .ndo_set_features       = hns3_nic_set_features,
1279         .ndo_get_stats64        = hns3_nic_get_stats64,
1280         .ndo_setup_tc           = hns3_nic_setup_tc,
1281         .ndo_set_rx_mode        = hns3_nic_set_rx_mode,
1282         .ndo_udp_tunnel_add     = hns3_nic_udp_tunnel_add,
1283         .ndo_udp_tunnel_del     = hns3_nic_udp_tunnel_del,
1284         .ndo_vlan_rx_add_vid    = hns3_vlan_rx_add_vid,
1285         .ndo_vlan_rx_kill_vid   = hns3_vlan_rx_kill_vid,
1286         .ndo_set_vf_vlan        = hns3_ndo_set_vf_vlan,
1287 };
1288
1289 /* hns3_probe - Device initialization routine
1290  * @pdev: PCI device information struct
1291  * @ent: entry in hns3_pci_tbl
1292  *
1293  * hns3_probe initializes a PF identified by a pci_dev structure.
1294  * The OS initialization, configuring of the PF private structure,
1295  * and a hardware reset occur.
1296  *
1297  * Returns 0 on success, negative on failure
1298  */
1299 static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1300 {
1301         struct hnae3_ae_dev *ae_dev;
1302         int ret;
1303
1304         ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1305                               GFP_KERNEL);
1306         if (!ae_dev) {
1307                 ret = -ENOMEM;
1308                 return ret;
1309         }
1310
1311         ae_dev->pdev = pdev;
1312         ae_dev->dev_type = HNAE3_DEV_KNIC;
1313         pci_set_drvdata(pdev, ae_dev);
1314
1315         return hnae3_register_ae_dev(ae_dev);
1316 }
1317
1318 /* hns3_remove - Device removal routine
1319  * @pdev: PCI device information struct
1320  */
1321 static void hns3_remove(struct pci_dev *pdev)
1322 {
1323         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1324
1325         hnae3_unregister_ae_dev(ae_dev);
1326
1327         devm_kfree(&pdev->dev, ae_dev);
1328
1329         pci_set_drvdata(pdev, NULL);
1330 }
1331
1332 static struct pci_driver hns3_driver = {
1333         .name     = hns3_driver_name,
1334         .id_table = hns3_pci_tbl,
1335         .probe    = hns3_probe,
1336         .remove   = hns3_remove,
1337 };
1338
1339 /* set default feature to hns3 */
1340 static void hns3_set_default_feature(struct net_device *netdev)
1341 {
1342         netdev->priv_flags |= IFF_UNICAST_FLT;
1343
1344         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1345                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1346                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1347                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1348                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1349
1350         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1351
1352         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1353
1354         netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1355                 NETIF_F_HW_VLAN_CTAG_FILTER |
1356                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1357                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1358                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1359                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1360
1361         netdev->vlan_features |=
1362                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1363                 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1364                 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1365                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1366                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1367
1368         netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1369                 NETIF_F_HW_VLAN_CTAG_FILTER |
1370                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1371                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1372                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1373                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1374 }
1375
1376 static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1377                              struct hns3_desc_cb *cb)
1378 {
1379         unsigned int order = hnae_page_order(ring);
1380         struct page *p;
1381
1382         p = dev_alloc_pages(order);
1383         if (!p)
1384                 return -ENOMEM;
1385
1386         cb->priv = p;
1387         cb->page_offset = 0;
1388         cb->reuse_flag = 0;
1389         cb->buf  = page_address(p);
1390         cb->length = hnae_page_size(ring);
1391         cb->type = DESC_TYPE_PAGE;
1392
1393         memset(cb->buf, 0, cb->length);
1394
1395         return 0;
1396 }
1397
1398 static void hns3_free_buffer(struct hns3_enet_ring *ring,
1399                              struct hns3_desc_cb *cb)
1400 {
1401         if (cb->type == DESC_TYPE_SKB)
1402                 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1403         else if (!HNAE3_IS_TX_RING(ring))
1404                 put_page((struct page *)cb->priv);
1405         memset(cb, 0, sizeof(*cb));
1406 }
1407
1408 static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1409 {
1410         cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1411                                cb->length, ring_to_dma_dir(ring));
1412
1413         if (dma_mapping_error(ring_to_dev(ring), cb->dma))
1414                 return -EIO;
1415
1416         return 0;
1417 }
1418
1419 static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1420                               struct hns3_desc_cb *cb)
1421 {
1422         if (cb->type == DESC_TYPE_SKB)
1423                 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1424                                  ring_to_dma_dir(ring));
1425         else
1426                 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1427                                ring_to_dma_dir(ring));
1428 }
1429
1430 static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1431 {
1432         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1433         ring->desc[i].addr = 0;
1434 }
1435
1436 static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1437 {
1438         struct hns3_desc_cb *cb = &ring->desc_cb[i];
1439
1440         if (!ring->desc_cb[i].dma)
1441                 return;
1442
1443         hns3_buffer_detach(ring, i);
1444         hns3_free_buffer(ring, cb);
1445 }
1446
1447 static void hns3_free_buffers(struct hns3_enet_ring *ring)
1448 {
1449         int i;
1450
1451         for (i = 0; i < ring->desc_num; i++)
1452                 hns3_free_buffer_detach(ring, i);
1453 }
1454
1455 /* free desc along with its attached buffer */
1456 static void hns3_free_desc(struct hns3_enet_ring *ring)
1457 {
1458         hns3_free_buffers(ring);
1459
1460         dma_unmap_single(ring_to_dev(ring), ring->desc_dma_addr,
1461                          ring->desc_num * sizeof(ring->desc[0]),
1462                          DMA_BIDIRECTIONAL);
1463         ring->desc_dma_addr = 0;
1464         kfree(ring->desc);
1465         ring->desc = NULL;
1466 }
1467
1468 static int hns3_alloc_desc(struct hns3_enet_ring *ring)
1469 {
1470         int size = ring->desc_num * sizeof(ring->desc[0]);
1471
1472         ring->desc = kzalloc(size, GFP_KERNEL);
1473         if (!ring->desc)
1474                 return -ENOMEM;
1475
1476         ring->desc_dma_addr = dma_map_single(ring_to_dev(ring), ring->desc,
1477                                              size, DMA_BIDIRECTIONAL);
1478         if (dma_mapping_error(ring_to_dev(ring), ring->desc_dma_addr)) {
1479                 ring->desc_dma_addr = 0;
1480                 kfree(ring->desc);
1481                 ring->desc = NULL;
1482                 return -ENOMEM;
1483         }
1484
1485         return 0;
1486 }
1487
1488 static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
1489                                    struct hns3_desc_cb *cb)
1490 {
1491         int ret;
1492
1493         ret = hns3_alloc_buffer(ring, cb);
1494         if (ret)
1495                 goto out;
1496
1497         ret = hns3_map_buffer(ring, cb);
1498         if (ret)
1499                 goto out_with_buf;
1500
1501         return 0;
1502
1503 out_with_buf:
1504         hns3_free_buffers(ring);
1505 out:
1506         return ret;
1507 }
1508
1509 static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
1510 {
1511         int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
1512
1513         if (ret)
1514                 return ret;
1515
1516         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1517
1518         return 0;
1519 }
1520
1521 /* Allocate memory for raw pkg, and map with dma */
1522 static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
1523 {
1524         int i, j, ret;
1525
1526         for (i = 0; i < ring->desc_num; i++) {
1527                 ret = hns3_alloc_buffer_attach(ring, i);
1528                 if (ret)
1529                         goto out_buffer_fail;
1530         }
1531
1532         return 0;
1533
1534 out_buffer_fail:
1535         for (j = i - 1; j >= 0; j--)
1536                 hns3_free_buffer_detach(ring, j);
1537         return ret;
1538 }
1539
1540 /* detach a in-used buffer and replace with a reserved one  */
1541 static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
1542                                 struct hns3_desc_cb *res_cb)
1543 {
1544         hns3_map_buffer(ring, &ring->desc_cb[i]);
1545         ring->desc_cb[i] = *res_cb;
1546         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1547 }
1548
1549 static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
1550 {
1551         ring->desc_cb[i].reuse_flag = 0;
1552         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
1553                 + ring->desc_cb[i].page_offset);
1554 }
1555
1556 static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
1557                                       int *pkts)
1558 {
1559         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
1560
1561         (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
1562         (*bytes) += desc_cb->length;
1563         /* desc_cb will be cleaned, after hnae_free_buffer_detach*/
1564         hns3_free_buffer_detach(ring, ring->next_to_clean);
1565
1566         ring_ptr_move_fw(ring, next_to_clean);
1567 }
1568
1569 static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
1570 {
1571         int u = ring->next_to_use;
1572         int c = ring->next_to_clean;
1573
1574         if (unlikely(h > ring->desc_num))
1575                 return 0;
1576
1577         return u > c ? (h > c && h <= u) : (h > c || h <= u);
1578 }
1579
1580 int hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
1581 {
1582         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1583         struct netdev_queue *dev_queue;
1584         int bytes, pkts;
1585         int head;
1586
1587         head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
1588         rmb(); /* Make sure head is ready before touch any data */
1589
1590         if (is_ring_empty(ring) || head == ring->next_to_clean)
1591                 return 0; /* no data to poll */
1592
1593         if (!is_valid_clean_head(ring, head)) {
1594                 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
1595                            ring->next_to_use, ring->next_to_clean);
1596
1597                 u64_stats_update_begin(&ring->syncp);
1598                 ring->stats.io_err_cnt++;
1599                 u64_stats_update_end(&ring->syncp);
1600                 return -EIO;
1601         }
1602
1603         bytes = 0;
1604         pkts = 0;
1605         while (head != ring->next_to_clean && budget) {
1606                 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
1607                 /* Issue prefetch for next Tx descriptor */
1608                 prefetch(&ring->desc_cb[ring->next_to_clean]);
1609                 budget--;
1610         }
1611
1612         ring->tqp_vector->tx_group.total_bytes += bytes;
1613         ring->tqp_vector->tx_group.total_packets += pkts;
1614
1615         u64_stats_update_begin(&ring->syncp);
1616         ring->stats.tx_bytes += bytes;
1617         ring->stats.tx_pkts += pkts;
1618         u64_stats_update_end(&ring->syncp);
1619
1620         dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
1621         netdev_tx_completed_queue(dev_queue, pkts, bytes);
1622
1623         if (unlikely(pkts && netif_carrier_ok(netdev) &&
1624                      (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
1625                 /* Make sure that anybody stopping the queue after this
1626                  * sees the new next_to_clean.
1627                  */
1628                 smp_mb();
1629                 if (netif_tx_queue_stopped(dev_queue)) {
1630                         netif_tx_wake_queue(dev_queue);
1631                         ring->stats.restart_queue++;
1632                 }
1633         }
1634
1635         return !!budget;
1636 }
1637
1638 static int hns3_desc_unused(struct hns3_enet_ring *ring)
1639 {
1640         int ntc = ring->next_to_clean;
1641         int ntu = ring->next_to_use;
1642
1643         return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
1644 }
1645
1646 static void
1647 hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
1648 {
1649         struct hns3_desc_cb *desc_cb;
1650         struct hns3_desc_cb res_cbs;
1651         int i, ret;
1652
1653         for (i = 0; i < cleand_count; i++) {
1654                 desc_cb = &ring->desc_cb[ring->next_to_use];
1655                 if (desc_cb->reuse_flag) {
1656                         u64_stats_update_begin(&ring->syncp);
1657                         ring->stats.reuse_pg_cnt++;
1658                         u64_stats_update_end(&ring->syncp);
1659
1660                         hns3_reuse_buffer(ring, ring->next_to_use);
1661                 } else {
1662                         ret = hns3_reserve_buffer_map(ring, &res_cbs);
1663                         if (ret) {
1664                                 u64_stats_update_begin(&ring->syncp);
1665                                 ring->stats.sw_err_cnt++;
1666                                 u64_stats_update_end(&ring->syncp);
1667
1668                                 netdev_err(ring->tqp->handle->kinfo.netdev,
1669                                            "hnae reserve buffer map failed.\n");
1670                                 break;
1671                         }
1672                         hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
1673                 }
1674
1675                 ring_ptr_move_fw(ring, next_to_use);
1676         }
1677
1678         wmb(); /* Make all data has been write before submit */
1679         writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
1680 }
1681
1682 /* hns3_nic_get_headlen - determine size of header for LRO/GRO
1683  * @data: pointer to the start of the headers
1684  * @max: total length of section to find headers in
1685  *
1686  * This function is meant to determine the length of headers that will
1687  * be recognized by hardware for LRO, GRO, and RSC offloads.  The main
1688  * motivation of doing this is to only perform one pull for IPv4 TCP
1689  * packets so that we can do basic things like calculating the gso_size
1690  * based on the average data per packet.
1691  */
1692 static unsigned int hns3_nic_get_headlen(unsigned char *data, u32 flag,
1693                                          unsigned int max_size)
1694 {
1695         unsigned char *network;
1696         u8 hlen;
1697
1698         /* This should never happen, but better safe than sorry */
1699         if (max_size < ETH_HLEN)
1700                 return max_size;
1701
1702         /* Initialize network frame pointer */
1703         network = data;
1704
1705         /* Set first protocol and move network header forward */
1706         network += ETH_HLEN;
1707
1708         /* Handle any vlan tag if present */
1709         if (hnae_get_field(flag, HNS3_RXD_VLAN_M, HNS3_RXD_VLAN_S)
1710                 == HNS3_RX_FLAG_VLAN_PRESENT) {
1711                 if ((typeof(max_size))(network - data) > (max_size - VLAN_HLEN))
1712                         return max_size;
1713
1714                 network += VLAN_HLEN;
1715         }
1716
1717         /* Handle L3 protocols */
1718         if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
1719                 == HNS3_RX_FLAG_L3ID_IPV4) {
1720                 if ((typeof(max_size))(network - data) >
1721                     (max_size - sizeof(struct iphdr)))
1722                         return max_size;
1723
1724                 /* Access ihl as a u8 to avoid unaligned access on ia64 */
1725                 hlen = (network[0] & 0x0F) << 2;
1726
1727                 /* Verify hlen meets minimum size requirements */
1728                 if (hlen < sizeof(struct iphdr))
1729                         return network - data;
1730
1731                 /* Record next protocol if header is present */
1732         } else if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
1733                 == HNS3_RX_FLAG_L3ID_IPV6) {
1734                 if ((typeof(max_size))(network - data) >
1735                     (max_size - sizeof(struct ipv6hdr)))
1736                         return max_size;
1737
1738                 /* Record next protocol */
1739                 hlen = sizeof(struct ipv6hdr);
1740         } else {
1741                 return network - data;
1742         }
1743
1744         /* Relocate pointer to start of L4 header */
1745         network += hlen;
1746
1747         /* Finally sort out TCP/UDP */
1748         if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
1749                 == HNS3_RX_FLAG_L4ID_TCP) {
1750                 if ((typeof(max_size))(network - data) >
1751                     (max_size - sizeof(struct tcphdr)))
1752                         return max_size;
1753
1754                 /* Access doff as a u8 to avoid unaligned access on ia64 */
1755                 hlen = (network[12] & 0xF0) >> 2;
1756
1757                 /* Verify hlen meets minimum size requirements */
1758                 if (hlen < sizeof(struct tcphdr))
1759                         return network - data;
1760
1761                 network += hlen;
1762         } else if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
1763                 == HNS3_RX_FLAG_L4ID_UDP) {
1764                 if ((typeof(max_size))(network - data) >
1765                     (max_size - sizeof(struct udphdr)))
1766                         return max_size;
1767
1768                 network += sizeof(struct udphdr);
1769         }
1770
1771         /* If everything has gone correctly network should be the
1772          * data section of the packet and will be the end of the header.
1773          * If not then it probably represents the end of the last recognized
1774          * header.
1775          */
1776         if ((typeof(max_size))(network - data) < max_size)
1777                 return network - data;
1778         else
1779                 return max_size;
1780 }
1781
1782 static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
1783                                 struct hns3_enet_ring *ring, int pull_len,
1784                                 struct hns3_desc_cb *desc_cb)
1785 {
1786         struct hns3_desc *desc;
1787         int truesize, size;
1788         int last_offset;
1789         bool twobufs;
1790
1791         twobufs = ((PAGE_SIZE < 8192) &&
1792                 hnae_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
1793
1794         desc = &ring->desc[ring->next_to_clean];
1795         size = le16_to_cpu(desc->rx.size);
1796
1797         if (twobufs) {
1798                 truesize = hnae_buf_size(ring);
1799         } else {
1800                 truesize = ALIGN(size, L1_CACHE_BYTES);
1801                 last_offset = hnae_page_size(ring) - hnae_buf_size(ring);
1802         }
1803
1804         skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
1805                         size - pull_len, truesize - pull_len);
1806
1807          /* Avoid re-using remote pages,flag default unreuse */
1808         if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
1809                 return;
1810
1811         if (twobufs) {
1812                 /* If we are only owner of page we can reuse it */
1813                 if (likely(page_count(desc_cb->priv) == 1)) {
1814                         /* Flip page offset to other buffer */
1815                         desc_cb->page_offset ^= truesize;
1816
1817                         desc_cb->reuse_flag = 1;
1818                         /* bump ref count on page before it is given*/
1819                         get_page(desc_cb->priv);
1820                 }
1821                 return;
1822         }
1823
1824         /* Move offset up to the next cache line */
1825         desc_cb->page_offset += truesize;
1826
1827         if (desc_cb->page_offset <= last_offset) {
1828                 desc_cb->reuse_flag = 1;
1829                 /* Bump ref count on page before it is given*/
1830                 get_page(desc_cb->priv);
1831         }
1832 }
1833
1834 static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
1835                              struct hns3_desc *desc)
1836 {
1837         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1838         int l3_type, l4_type;
1839         u32 bd_base_info;
1840         int ol4_type;
1841         u32 l234info;
1842
1843         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
1844         l234info = le32_to_cpu(desc->rx.l234_info);
1845
1846         skb->ip_summed = CHECKSUM_NONE;
1847
1848         skb_checksum_none_assert(skb);
1849
1850         if (!(netdev->features & NETIF_F_RXCSUM))
1851                 return;
1852
1853         /* check if hardware has done checksum */
1854         if (!hnae_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
1855                 return;
1856
1857         if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L3E_B) ||
1858                      hnae_get_bit(l234info, HNS3_RXD_L4E_B) ||
1859                      hnae_get_bit(l234info, HNS3_RXD_OL3E_B) ||
1860                      hnae_get_bit(l234info, HNS3_RXD_OL4E_B))) {
1861                 netdev_err(netdev, "L3/L4 error pkt\n");
1862                 u64_stats_update_begin(&ring->syncp);
1863                 ring->stats.l3l4_csum_err++;
1864                 u64_stats_update_end(&ring->syncp);
1865
1866                 return;
1867         }
1868
1869         l3_type = hnae_get_field(l234info, HNS3_RXD_L3ID_M,
1870                                  HNS3_RXD_L3ID_S);
1871         l4_type = hnae_get_field(l234info, HNS3_RXD_L4ID_M,
1872                                  HNS3_RXD_L4ID_S);
1873
1874         ol4_type = hnae_get_field(l234info, HNS3_RXD_OL4ID_M, HNS3_RXD_OL4ID_S);
1875         switch (ol4_type) {
1876         case HNS3_OL4_TYPE_MAC_IN_UDP:
1877         case HNS3_OL4_TYPE_NVGRE:
1878                 skb->csum_level = 1;
1879         case HNS3_OL4_TYPE_NO_TUN:
1880                 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
1881                 if (l3_type == HNS3_L3_TYPE_IPV4 ||
1882                     (l3_type == HNS3_L3_TYPE_IPV6 &&
1883                      (l4_type == HNS3_L4_TYPE_UDP ||
1884                       l4_type == HNS3_L4_TYPE_TCP ||
1885                       l4_type == HNS3_L4_TYPE_SCTP)))
1886                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1887                 break;
1888         }
1889 }
1890
1891 static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
1892                              struct sk_buff **out_skb, int *out_bnum)
1893 {
1894         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1895         struct hns3_desc_cb *desc_cb;
1896         struct hns3_desc *desc;
1897         struct sk_buff *skb;
1898         unsigned char *va;
1899         u32 bd_base_info;
1900         int pull_len;
1901         u32 l234info;
1902         int length;
1903         int bnum;
1904
1905         desc = &ring->desc[ring->next_to_clean];
1906         desc_cb = &ring->desc_cb[ring->next_to_clean];
1907
1908         prefetch(desc);
1909
1910         length = le16_to_cpu(desc->rx.pkt_len);
1911         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
1912         l234info = le32_to_cpu(desc->rx.l234_info);
1913
1914         /* Check valid BD */
1915         if (!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))
1916                 return -EFAULT;
1917
1918         va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
1919
1920         /* Prefetch first cache line of first page
1921          * Idea is to cache few bytes of the header of the packet. Our L1 Cache
1922          * line size is 64B so need to prefetch twice to make it 128B. But in
1923          * actual we can have greater size of caches with 128B Level 1 cache
1924          * lines. In such a case, single fetch would suffice to cache in the
1925          * relevant part of the header.
1926          */
1927         prefetch(va);
1928 #if L1_CACHE_BYTES < 128
1929         prefetch(va + L1_CACHE_BYTES);
1930 #endif
1931
1932         skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
1933                                         HNS3_RX_HEAD_SIZE);
1934         if (unlikely(!skb)) {
1935                 netdev_err(netdev, "alloc rx skb fail\n");
1936
1937                 u64_stats_update_begin(&ring->syncp);
1938                 ring->stats.sw_err_cnt++;
1939                 u64_stats_update_end(&ring->syncp);
1940
1941                 return -ENOMEM;
1942         }
1943
1944         prefetchw(skb->data);
1945
1946         bnum = 1;
1947         if (length <= HNS3_RX_HEAD_SIZE) {
1948                 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
1949
1950                 /* We can reuse buffer as-is, just make sure it is local */
1951                 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
1952                         desc_cb->reuse_flag = 1;
1953                 else /* This page cannot be reused so discard it */
1954                         put_page(desc_cb->priv);
1955
1956                 ring_ptr_move_fw(ring, next_to_clean);
1957         } else {
1958                 u64_stats_update_begin(&ring->syncp);
1959                 ring->stats.seg_pkt_cnt++;
1960                 u64_stats_update_end(&ring->syncp);
1961
1962                 pull_len = hns3_nic_get_headlen(va, l234info,
1963                                                 HNS3_RX_HEAD_SIZE);
1964                 memcpy(__skb_put(skb, pull_len), va,
1965                        ALIGN(pull_len, sizeof(long)));
1966
1967                 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
1968                 ring_ptr_move_fw(ring, next_to_clean);
1969
1970                 while (!hnae_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
1971                         desc = &ring->desc[ring->next_to_clean];
1972                         desc_cb = &ring->desc_cb[ring->next_to_clean];
1973                         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
1974                         hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
1975                         ring_ptr_move_fw(ring, next_to_clean);
1976                         bnum++;
1977                 }
1978         }
1979
1980         *out_bnum = bnum;
1981
1982         if (unlikely(!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
1983                 netdev_err(netdev, "no valid bd,%016llx,%016llx\n",
1984                            ((u64 *)desc)[0], ((u64 *)desc)[1]);
1985                 u64_stats_update_begin(&ring->syncp);
1986                 ring->stats.non_vld_descs++;
1987                 u64_stats_update_end(&ring->syncp);
1988
1989                 dev_kfree_skb_any(skb);
1990                 return -EINVAL;
1991         }
1992
1993         if (unlikely((!desc->rx.pkt_len) ||
1994                      hnae_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
1995                 netdev_err(netdev, "truncated pkt\n");
1996                 u64_stats_update_begin(&ring->syncp);
1997                 ring->stats.err_pkt_len++;
1998                 u64_stats_update_end(&ring->syncp);
1999
2000                 dev_kfree_skb_any(skb);
2001                 return -EFAULT;
2002         }
2003
2004         if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L2E_B))) {
2005                 netdev_err(netdev, "L2 error pkt\n");
2006                 u64_stats_update_begin(&ring->syncp);
2007                 ring->stats.l2_err++;
2008                 u64_stats_update_end(&ring->syncp);
2009
2010                 dev_kfree_skb_any(skb);
2011                 return -EFAULT;
2012         }
2013
2014         u64_stats_update_begin(&ring->syncp);
2015         ring->stats.rx_pkts++;
2016         ring->stats.rx_bytes += skb->len;
2017         u64_stats_update_end(&ring->syncp);
2018
2019         ring->tqp_vector->rx_group.total_bytes += skb->len;
2020
2021         hns3_rx_checksum(ring, skb, desc);
2022         return 0;
2023 }
2024
2025 static int hns3_clean_rx_ring(struct hns3_enet_ring *ring, int budget)
2026 {
2027 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2028         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2029         int recv_pkts, recv_bds, clean_count, err;
2030         int unused_count = hns3_desc_unused(ring);
2031         struct sk_buff *skb = NULL;
2032         int num, bnum = 0;
2033
2034         num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2035         rmb(); /* Make sure num taken effect before the other data is touched */
2036
2037         recv_pkts = 0, recv_bds = 0, clean_count = 0;
2038         num -= unused_count;
2039
2040         while (recv_pkts < budget && recv_bds < num) {
2041                 /* Reuse or realloc buffers */
2042                 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2043                         hns3_nic_alloc_rx_buffers(ring,
2044                                                   clean_count + unused_count);
2045                         clean_count = 0;
2046                         unused_count = hns3_desc_unused(ring);
2047                 }
2048
2049                 /* Poll one pkt */
2050                 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2051                 if (unlikely(!skb)) /* This fault cannot be repaired */
2052                         goto out;
2053
2054                 recv_bds += bnum;
2055                 clean_count += bnum;
2056                 if (unlikely(err)) {  /* Do jump the err */
2057                         recv_pkts++;
2058                         continue;
2059                 }
2060
2061                 /* Do update ip stack process */
2062                 skb->protocol = eth_type_trans(skb, netdev);
2063                 (void)napi_gro_receive(&ring->tqp_vector->napi, skb);
2064
2065                 recv_pkts++;
2066         }
2067
2068 out:
2069         /* Make all data has been write before submit */
2070         if (clean_count + unused_count > 0)
2071                 hns3_nic_alloc_rx_buffers(ring,
2072                                           clean_count + unused_count);
2073
2074         return recv_pkts;
2075 }
2076
2077 static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2078 {
2079 #define HNS3_RX_ULTRA_PACKET_RATE 40000
2080         enum hns3_flow_level_range new_flow_level;
2081         struct hns3_enet_tqp_vector *tqp_vector;
2082         int packets_per_secs;
2083         int bytes_per_usecs;
2084         u16 new_int_gl;
2085         int usecs;
2086
2087         if (!ring_group->int_gl)
2088                 return false;
2089
2090         if (ring_group->total_packets == 0) {
2091                 ring_group->int_gl = HNS3_INT_GL_50K;
2092                 ring_group->flow_level = HNS3_FLOW_LOW;
2093                 return true;
2094         }
2095
2096         /* Simple throttlerate management
2097          * 0-10MB/s   lower     (50000 ints/s)
2098          * 10-20MB/s   middle    (20000 ints/s)
2099          * 20-1249MB/s high      (18000 ints/s)
2100          * > 40000pps  ultra     (8000 ints/s)
2101          */
2102         new_flow_level = ring_group->flow_level;
2103         new_int_gl = ring_group->int_gl;
2104         tqp_vector = ring_group->ring->tqp_vector;
2105         usecs = (ring_group->int_gl << 1);
2106         bytes_per_usecs = ring_group->total_bytes / usecs;
2107         /* 1000000 microseconds */
2108         packets_per_secs = ring_group->total_packets * 1000000 / usecs;
2109
2110         switch (new_flow_level) {
2111         case HNS3_FLOW_LOW:
2112                 if (bytes_per_usecs > 10)
2113                         new_flow_level = HNS3_FLOW_MID;
2114                 break;
2115         case HNS3_FLOW_MID:
2116                 if (bytes_per_usecs > 20)
2117                         new_flow_level = HNS3_FLOW_HIGH;
2118                 else if (bytes_per_usecs <= 10)
2119                         new_flow_level = HNS3_FLOW_LOW;
2120                 break;
2121         case HNS3_FLOW_HIGH:
2122         case HNS3_FLOW_ULTRA:
2123         default:
2124                 if (bytes_per_usecs <= 20)
2125                         new_flow_level = HNS3_FLOW_MID;
2126                 break;
2127         }
2128
2129         if ((packets_per_secs > HNS3_RX_ULTRA_PACKET_RATE) &&
2130             (&tqp_vector->rx_group == ring_group))
2131                 new_flow_level = HNS3_FLOW_ULTRA;
2132
2133         switch (new_flow_level) {
2134         case HNS3_FLOW_LOW:
2135                 new_int_gl = HNS3_INT_GL_50K;
2136                 break;
2137         case HNS3_FLOW_MID:
2138                 new_int_gl = HNS3_INT_GL_20K;
2139                 break;
2140         case HNS3_FLOW_HIGH:
2141                 new_int_gl = HNS3_INT_GL_18K;
2142                 break;
2143         case HNS3_FLOW_ULTRA:
2144                 new_int_gl = HNS3_INT_GL_8K;
2145                 break;
2146         default:
2147                 break;
2148         }
2149
2150         ring_group->total_bytes = 0;
2151         ring_group->total_packets = 0;
2152         ring_group->flow_level = new_flow_level;
2153         if (new_int_gl != ring_group->int_gl) {
2154                 ring_group->int_gl = new_int_gl;
2155                 return true;
2156         }
2157         return false;
2158 }
2159
2160 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2161 {
2162         u16 rx_int_gl, tx_int_gl;
2163         bool rx, tx;
2164
2165         rx = hns3_get_new_int_gl(&tqp_vector->rx_group);
2166         tx = hns3_get_new_int_gl(&tqp_vector->tx_group);
2167         rx_int_gl = tqp_vector->rx_group.int_gl;
2168         tx_int_gl = tqp_vector->tx_group.int_gl;
2169         if (rx && tx) {
2170                 if (rx_int_gl > tx_int_gl) {
2171                         tqp_vector->tx_group.int_gl = rx_int_gl;
2172                         tqp_vector->tx_group.flow_level =
2173                                 tqp_vector->rx_group.flow_level;
2174                         hns3_set_vector_coalesc_gl(tqp_vector, rx_int_gl);
2175                 } else {
2176                         tqp_vector->rx_group.int_gl = tx_int_gl;
2177                         tqp_vector->rx_group.flow_level =
2178                                 tqp_vector->tx_group.flow_level;
2179                         hns3_set_vector_coalesc_gl(tqp_vector, tx_int_gl);
2180                 }
2181         }
2182 }
2183
2184 static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2185 {
2186         struct hns3_enet_ring *ring;
2187         int rx_pkt_total = 0;
2188
2189         struct hns3_enet_tqp_vector *tqp_vector =
2190                 container_of(napi, struct hns3_enet_tqp_vector, napi);
2191         bool clean_complete = true;
2192         int rx_budget;
2193
2194         /* Since the actual Tx work is minimal, we can give the Tx a larger
2195          * budget and be more aggressive about cleaning up the Tx descriptors.
2196          */
2197         hns3_for_each_ring(ring, tqp_vector->tx_group) {
2198                 if (!hns3_clean_tx_ring(ring, budget))
2199                         clean_complete = false;
2200         }
2201
2202         /* make sure rx ring budget not smaller than 1 */
2203         rx_budget = max(budget / tqp_vector->num_tqps, 1);
2204
2205         hns3_for_each_ring(ring, tqp_vector->rx_group) {
2206                 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget);
2207
2208                 if (rx_cleaned >= rx_budget)
2209                         clean_complete = false;
2210
2211                 rx_pkt_total += rx_cleaned;
2212         }
2213
2214         tqp_vector->rx_group.total_packets += rx_pkt_total;
2215
2216         if (!clean_complete)
2217                 return budget;
2218
2219         napi_complete(napi);
2220         hns3_update_new_int_gl(tqp_vector);
2221         hns3_mask_vector_irq(tqp_vector, 1);
2222
2223         return rx_pkt_total;
2224 }
2225
2226 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2227                                       struct hnae3_ring_chain_node *head)
2228 {
2229         struct pci_dev *pdev = tqp_vector->handle->pdev;
2230         struct hnae3_ring_chain_node *cur_chain = head;
2231         struct hnae3_ring_chain_node *chain;
2232         struct hns3_enet_ring *tx_ring;
2233         struct hns3_enet_ring *rx_ring;
2234
2235         tx_ring = tqp_vector->tx_group.ring;
2236         if (tx_ring) {
2237                 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2238                 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2239                              HNAE3_RING_TYPE_TX);
2240
2241                 cur_chain->next = NULL;
2242
2243                 while (tx_ring->next) {
2244                         tx_ring = tx_ring->next;
2245
2246                         chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2247                                              GFP_KERNEL);
2248                         if (!chain)
2249                                 return -ENOMEM;
2250
2251                         cur_chain->next = chain;
2252                         chain->tqp_index = tx_ring->tqp->tqp_index;
2253                         hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2254                                      HNAE3_RING_TYPE_TX);
2255
2256                         cur_chain = chain;
2257                 }
2258         }
2259
2260         rx_ring = tqp_vector->rx_group.ring;
2261         if (!tx_ring && rx_ring) {
2262                 cur_chain->next = NULL;
2263                 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2264                 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2265                              HNAE3_RING_TYPE_RX);
2266
2267                 rx_ring = rx_ring->next;
2268         }
2269
2270         while (rx_ring) {
2271                 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2272                 if (!chain)
2273                         return -ENOMEM;
2274
2275                 cur_chain->next = chain;
2276                 chain->tqp_index = rx_ring->tqp->tqp_index;
2277                 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2278                              HNAE3_RING_TYPE_RX);
2279                 cur_chain = chain;
2280
2281                 rx_ring = rx_ring->next;
2282         }
2283
2284         return 0;
2285 }
2286
2287 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2288                                         struct hnae3_ring_chain_node *head)
2289 {
2290         struct pci_dev *pdev = tqp_vector->handle->pdev;
2291         struct hnae3_ring_chain_node *chain_tmp, *chain;
2292
2293         chain = head->next;
2294
2295         while (chain) {
2296                 chain_tmp = chain->next;
2297                 devm_kfree(&pdev->dev, chain);
2298                 chain = chain_tmp;
2299         }
2300 }
2301
2302 static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2303                                    struct hns3_enet_ring *ring)
2304 {
2305         ring->next = group->ring;
2306         group->ring = ring;
2307
2308         group->count++;
2309 }
2310
2311 static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2312 {
2313         struct hnae3_ring_chain_node vector_ring_chain;
2314         struct hnae3_handle *h = priv->ae_handle;
2315         struct hns3_enet_tqp_vector *tqp_vector;
2316         struct hnae3_vector_info *vector;
2317         struct pci_dev *pdev = h->pdev;
2318         u16 tqp_num = h->kinfo.num_tqps;
2319         u16 vector_num;
2320         int ret = 0;
2321         u16 i;
2322
2323         /* RSS size, cpu online and vector_num should be the same */
2324         /* Should consider 2p/4p later */
2325         vector_num = min_t(u16, num_online_cpus(), tqp_num);
2326         vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2327                               GFP_KERNEL);
2328         if (!vector)
2329                 return -ENOMEM;
2330
2331         vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2332
2333         priv->vector_num = vector_num;
2334         priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2335                 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2336                              GFP_KERNEL);
2337         if (!priv->tqp_vector)
2338                 return -ENOMEM;
2339
2340         for (i = 0; i < tqp_num; i++) {
2341                 u16 vector_i = i % vector_num;
2342
2343                 tqp_vector = &priv->tqp_vector[vector_i];
2344
2345                 hns3_add_ring_to_group(&tqp_vector->tx_group,
2346                                        priv->ring_data[i].ring);
2347
2348                 hns3_add_ring_to_group(&tqp_vector->rx_group,
2349                                        priv->ring_data[i + tqp_num].ring);
2350
2351                 tqp_vector->idx = vector_i;
2352                 tqp_vector->mask_addr = vector[vector_i].io_addr;
2353                 tqp_vector->vector_irq = vector[vector_i].vector;
2354                 tqp_vector->num_tqps++;
2355
2356                 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2357                 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
2358         }
2359
2360         for (i = 0; i < vector_num; i++) {
2361                 tqp_vector = &priv->tqp_vector[i];
2362
2363                 tqp_vector->rx_group.total_bytes = 0;
2364                 tqp_vector->rx_group.total_packets = 0;
2365                 tqp_vector->tx_group.total_bytes = 0;
2366                 tqp_vector->tx_group.total_packets = 0;
2367                 hns3_vector_gl_rl_init(tqp_vector);
2368                 tqp_vector->handle = h;
2369
2370                 ret = hns3_get_vector_ring_chain(tqp_vector,
2371                                                  &vector_ring_chain);
2372                 if (ret)
2373                         goto out;
2374
2375                 ret = h->ae_algo->ops->map_ring_to_vector(h,
2376                         tqp_vector->vector_irq, &vector_ring_chain);
2377                 if (ret)
2378                         goto out;
2379
2380                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2381
2382                 netif_napi_add(priv->netdev, &tqp_vector->napi,
2383                                hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2384         }
2385
2386 out:
2387         devm_kfree(&pdev->dev, vector);
2388         return ret;
2389 }
2390
2391 static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
2392 {
2393         struct hnae3_ring_chain_node vector_ring_chain;
2394         struct hnae3_handle *h = priv->ae_handle;
2395         struct hns3_enet_tqp_vector *tqp_vector;
2396         struct pci_dev *pdev = h->pdev;
2397         int i, ret;
2398
2399         for (i = 0; i < priv->vector_num; i++) {
2400                 tqp_vector = &priv->tqp_vector[i];
2401
2402                 ret = hns3_get_vector_ring_chain(tqp_vector,
2403                                                  &vector_ring_chain);
2404                 if (ret)
2405                         return ret;
2406
2407                 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
2408                         tqp_vector->vector_irq, &vector_ring_chain);
2409                 if (ret)
2410                         return ret;
2411
2412                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2413
2414                 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
2415                         (void)irq_set_affinity_hint(
2416                                 priv->tqp_vector[i].vector_irq,
2417                                                     NULL);
2418                         devm_free_irq(&pdev->dev,
2419                                       priv->tqp_vector[i].vector_irq,
2420                                       &priv->tqp_vector[i]);
2421                 }
2422
2423                 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
2424
2425                 netif_napi_del(&priv->tqp_vector[i].napi);
2426         }
2427
2428         devm_kfree(&pdev->dev, priv->tqp_vector);
2429
2430         return 0;
2431 }
2432
2433 static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
2434                              int ring_type)
2435 {
2436         struct hns3_nic_ring_data *ring_data = priv->ring_data;
2437         int queue_num = priv->ae_handle->kinfo.num_tqps;
2438         struct pci_dev *pdev = priv->ae_handle->pdev;
2439         struct hns3_enet_ring *ring;
2440
2441         ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
2442         if (!ring)
2443                 return -ENOMEM;
2444
2445         if (ring_type == HNAE3_RING_TYPE_TX) {
2446                 ring_data[q->tqp_index].ring = ring;
2447                 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
2448         } else {
2449                 ring_data[q->tqp_index + queue_num].ring = ring;
2450                 ring->io_base = q->io_base;
2451         }
2452
2453         hnae_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
2454
2455         ring_data[q->tqp_index].queue_index = q->tqp_index;
2456
2457         ring->tqp = q;
2458         ring->desc = NULL;
2459         ring->desc_cb = NULL;
2460         ring->dev = priv->dev;
2461         ring->desc_dma_addr = 0;
2462         ring->buf_size = q->buf_size;
2463         ring->desc_num = q->desc_num;
2464         ring->next_to_use = 0;
2465         ring->next_to_clean = 0;
2466
2467         return 0;
2468 }
2469
2470 static int hns3_queue_to_ring(struct hnae3_queue *tqp,
2471                               struct hns3_nic_priv *priv)
2472 {
2473         int ret;
2474
2475         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
2476         if (ret)
2477                 return ret;
2478
2479         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
2480         if (ret)
2481                 return ret;
2482
2483         return 0;
2484 }
2485
2486 static int hns3_get_ring_config(struct hns3_nic_priv *priv)
2487 {
2488         struct hnae3_handle *h = priv->ae_handle;
2489         struct pci_dev *pdev = h->pdev;
2490         int i, ret;
2491
2492         priv->ring_data =  devm_kzalloc(&pdev->dev, h->kinfo.num_tqps *
2493                                         sizeof(*priv->ring_data) * 2,
2494                                         GFP_KERNEL);
2495         if (!priv->ring_data)
2496                 return -ENOMEM;
2497
2498         for (i = 0; i < h->kinfo.num_tqps; i++) {
2499                 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
2500                 if (ret)
2501                         goto err;
2502         }
2503
2504         return 0;
2505 err:
2506         devm_kfree(&pdev->dev, priv->ring_data);
2507         return ret;
2508 }
2509
2510 static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
2511 {
2512         int ret;
2513
2514         if (ring->desc_num <= 0 || ring->buf_size <= 0)
2515                 return -EINVAL;
2516
2517         ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
2518                                 GFP_KERNEL);
2519         if (!ring->desc_cb) {
2520                 ret = -ENOMEM;
2521                 goto out;
2522         }
2523
2524         ret = hns3_alloc_desc(ring);
2525         if (ret)
2526                 goto out_with_desc_cb;
2527
2528         if (!HNAE3_IS_TX_RING(ring)) {
2529                 ret = hns3_alloc_ring_buffers(ring);
2530                 if (ret)
2531                         goto out_with_desc;
2532         }
2533
2534         return 0;
2535
2536 out_with_desc:
2537         hns3_free_desc(ring);
2538 out_with_desc_cb:
2539         kfree(ring->desc_cb);
2540         ring->desc_cb = NULL;
2541 out:
2542         return ret;
2543 }
2544
2545 static void hns3_fini_ring(struct hns3_enet_ring *ring)
2546 {
2547         hns3_free_desc(ring);
2548         kfree(ring->desc_cb);
2549         ring->desc_cb = NULL;
2550         ring->next_to_clean = 0;
2551         ring->next_to_use = 0;
2552 }
2553
2554 int hns3_buf_size2type(u32 buf_size)
2555 {
2556         int bd_size_type;
2557
2558         switch (buf_size) {
2559         case 512:
2560                 bd_size_type = HNS3_BD_SIZE_512_TYPE;
2561                 break;
2562         case 1024:
2563                 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
2564                 break;
2565         case 2048:
2566                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2567                 break;
2568         case 4096:
2569                 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
2570                 break;
2571         default:
2572                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2573         }
2574
2575         return bd_size_type;
2576 }
2577
2578 static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
2579 {
2580         dma_addr_t dma = ring->desc_dma_addr;
2581         struct hnae3_queue *q = ring->tqp;
2582
2583         if (!HNAE3_IS_TX_RING(ring)) {
2584                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
2585                                (u32)dma);
2586                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
2587                                (u32)((dma >> 31) >> 1));
2588
2589                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
2590                                hns3_buf_size2type(ring->buf_size));
2591                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
2592                                ring->desc_num / 8 - 1);
2593
2594         } else {
2595                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
2596                                (u32)dma);
2597                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
2598                                (u32)((dma >> 31) >> 1));
2599
2600                 hns3_write_dev(q, HNS3_RING_TX_RING_BD_LEN_REG,
2601                                hns3_buf_size2type(ring->buf_size));
2602                 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
2603                                ring->desc_num / 8 - 1);
2604         }
2605 }
2606
2607 static int hns3_init_all_ring(struct hns3_nic_priv *priv)
2608 {
2609         struct hnae3_handle *h = priv->ae_handle;
2610         int ring_num = h->kinfo.num_tqps * 2;
2611         int i, j;
2612         int ret;
2613
2614         for (i = 0; i < ring_num; i++) {
2615                 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
2616                 if (ret) {
2617                         dev_err(priv->dev,
2618                                 "Alloc ring memory fail! ret=%d\n", ret);
2619                         goto out_when_alloc_ring_memory;
2620                 }
2621
2622                 hns3_init_ring_hw(priv->ring_data[i].ring);
2623
2624                 u64_stats_init(&priv->ring_data[i].ring->syncp);
2625         }
2626
2627         return 0;
2628
2629 out_when_alloc_ring_memory:
2630         for (j = i - 1; j >= 0; j--)
2631                 hns3_fini_ring(priv->ring_data[i].ring);
2632
2633         return -ENOMEM;
2634 }
2635
2636 static int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
2637 {
2638         struct hnae3_handle *h = priv->ae_handle;
2639         int i;
2640
2641         for (i = 0; i < h->kinfo.num_tqps; i++) {
2642                 if (h->ae_algo->ops->reset_queue)
2643                         h->ae_algo->ops->reset_queue(h, i);
2644
2645                 hns3_fini_ring(priv->ring_data[i].ring);
2646                 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
2647         }
2648
2649         return 0;
2650 }
2651
2652 /* Set mac addr if it is configured. or leave it to the AE driver */
2653 static void hns3_init_mac_addr(struct net_device *netdev)
2654 {
2655         struct hns3_nic_priv *priv = netdev_priv(netdev);
2656         struct hnae3_handle *h = priv->ae_handle;
2657         u8 mac_addr_temp[ETH_ALEN];
2658
2659         if (h->ae_algo->ops->get_mac_addr) {
2660                 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
2661                 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
2662         }
2663
2664         /* Check if the MAC address is valid, if not get a random one */
2665         if (!is_valid_ether_addr(netdev->dev_addr)) {
2666                 eth_hw_addr_random(netdev);
2667                 dev_warn(priv->dev, "using random MAC address %pM\n",
2668                          netdev->dev_addr);
2669                 /* Also copy this new MAC address into hdev */
2670                 if (h->ae_algo->ops->set_mac_addr)
2671                         h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr);
2672         }
2673 }
2674
2675 static void hns3_nic_set_priv_ops(struct net_device *netdev)
2676 {
2677         struct hns3_nic_priv *priv = netdev_priv(netdev);
2678
2679         if ((netdev->features & NETIF_F_TSO) ||
2680             (netdev->features & NETIF_F_TSO6)) {
2681                 priv->ops.fill_desc = hns3_fill_desc_tso;
2682                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
2683         } else {
2684                 priv->ops.fill_desc = hns3_fill_desc;
2685                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
2686         }
2687 }
2688
2689 static int hns3_client_init(struct hnae3_handle *handle)
2690 {
2691         struct pci_dev *pdev = handle->pdev;
2692         struct hns3_nic_priv *priv;
2693         struct net_device *netdev;
2694         int ret;
2695
2696         netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv),
2697                                    handle->kinfo.num_tqps);
2698         if (!netdev)
2699                 return -ENOMEM;
2700
2701         priv = netdev_priv(netdev);
2702         priv->dev = &pdev->dev;
2703         priv->netdev = netdev;
2704         priv->ae_handle = handle;
2705
2706         handle->kinfo.netdev = netdev;
2707         handle->priv = (void *)priv;
2708
2709         hns3_init_mac_addr(netdev);
2710
2711         hns3_set_default_feature(netdev);
2712
2713         netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
2714         netdev->priv_flags |= IFF_UNICAST_FLT;
2715         netdev->netdev_ops = &hns3_nic_netdev_ops;
2716         SET_NETDEV_DEV(netdev, &pdev->dev);
2717         hns3_ethtool_set_ops(netdev);
2718         hns3_nic_set_priv_ops(netdev);
2719
2720         /* Carrier off reporting is important to ethtool even BEFORE open */
2721         netif_carrier_off(netdev);
2722
2723         ret = hns3_get_ring_config(priv);
2724         if (ret) {
2725                 ret = -ENOMEM;
2726                 goto out_get_ring_cfg;
2727         }
2728
2729         ret = hns3_nic_init_vector_data(priv);
2730         if (ret) {
2731                 ret = -ENOMEM;
2732                 goto out_init_vector_data;
2733         }
2734
2735         ret = hns3_init_all_ring(priv);
2736         if (ret) {
2737                 ret = -ENOMEM;
2738                 goto out_init_ring_data;
2739         }
2740
2741         ret = register_netdev(netdev);
2742         if (ret) {
2743                 dev_err(priv->dev, "probe register netdev fail!\n");
2744                 goto out_reg_netdev_fail;
2745         }
2746
2747         return ret;
2748
2749 out_reg_netdev_fail:
2750 out_init_ring_data:
2751         (void)hns3_nic_uninit_vector_data(priv);
2752         priv->ring_data = NULL;
2753 out_init_vector_data:
2754 out_get_ring_cfg:
2755         priv->ae_handle = NULL;
2756         free_netdev(netdev);
2757         return ret;
2758 }
2759
2760 static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
2761 {
2762         struct net_device *netdev = handle->kinfo.netdev;
2763         struct hns3_nic_priv *priv = netdev_priv(netdev);
2764         int ret;
2765
2766         if (netdev->reg_state != NETREG_UNINITIALIZED)
2767                 unregister_netdev(netdev);
2768
2769         ret = hns3_nic_uninit_vector_data(priv);
2770         if (ret)
2771                 netdev_err(netdev, "uninit vector error\n");
2772
2773         ret = hns3_uninit_all_ring(priv);
2774         if (ret)
2775                 netdev_err(netdev, "uninit ring error\n");
2776
2777         priv->ring_data = NULL;
2778
2779         free_netdev(netdev);
2780 }
2781
2782 static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
2783 {
2784         struct net_device *netdev = handle->kinfo.netdev;
2785
2786         if (!netdev)
2787                 return;
2788
2789         if (linkup) {
2790                 netif_carrier_on(netdev);
2791                 netif_tx_wake_all_queues(netdev);
2792                 netdev_info(netdev, "link up\n");
2793         } else {
2794                 netif_carrier_off(netdev);
2795                 netif_tx_stop_all_queues(netdev);
2796                 netdev_info(netdev, "link down\n");
2797         }
2798 }
2799
2800 const struct hnae3_client_ops client_ops = {
2801         .init_instance = hns3_client_init,
2802         .uninit_instance = hns3_client_uninit,
2803         .link_status_change = hns3_link_status_change,
2804 };
2805
2806 /* hns3_init_module - Driver registration routine
2807  * hns3_init_module is the first routine called when the driver is
2808  * loaded. All it does is register with the PCI subsystem.
2809  */
2810 static int __init hns3_init_module(void)
2811 {
2812         int ret;
2813
2814         pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
2815         pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
2816
2817         client.type = HNAE3_CLIENT_KNIC;
2818         snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
2819                  hns3_driver_name);
2820
2821         client.ops = &client_ops;
2822
2823         ret = hnae3_register_client(&client);
2824         if (ret)
2825                 return ret;
2826
2827         ret = pci_register_driver(&hns3_driver);
2828         if (ret)
2829                 hnae3_unregister_client(&client);
2830
2831         return ret;
2832 }
2833 module_init(hns3_init_module);
2834
2835 /* hns3_exit_module - Driver exit cleanup routine
2836  * hns3_exit_module is called just before the driver is removed
2837  * from memory.
2838  */
2839 static void __exit hns3_exit_module(void)
2840 {
2841         pci_unregister_driver(&hns3_driver);
2842         hnae3_unregister_client(&client);
2843 }
2844 module_exit(hns3_exit_module);
2845
2846 MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
2847 MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
2848 MODULE_LICENSE("GPL");
2849 MODULE_ALIAS("pci:hns-nic");