drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.c

   1 /*
   2  * Copyright (c) 2016~2017 Hisilicon Limited.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  */
   9
  10 #include <linux/dma-mapping.h>
  11 #include <linux/etherdevice.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/if_vlan.h>
  14 #include <linux/ip.h>
  15 #include <linux/ipv6.h>
  16 #include <linux/module.h>
  17 #include <linux/pci.h>
  18 #include <linux/skbuff.h>
  19 #include <linux/sctp.h>
  20 #include <linux/vermagic.h>
  21 #include <net/gre.h>
  22 #include <net/vxlan.h>
  23
  24 #include "hnae3.h"
  25 #include "hns3_enet.h"
  26
  27 const char hns3_driver_name[] = "hns3";
  28 const char hns3_driver_version[] = VERMAGIC_STRING;
  29 static const char hns3_driver_string[] =
  30                         "Hisilicon Ethernet Network Driver for Hip08 Family";
  31 static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
  32 static struct hnae3_client client;
  33
  34 /* hns3_pci_tbl - PCI Device ID Table
  35  *
  36  * Last entry must be all 0s
  37  *
  38  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  39  *   Class, Class Mask, private data (not used) }
  40  */
  41 static const struct pci_device_id hns3_pci_tbl[] = {
  42         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
  43         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
  44         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
  45         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
  46         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
  47         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
  48         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
  49         /* required last entry */
  50         {0, }
  51 };
  52 MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
  53
  54 static irqreturn_t hns3_irq_handle(int irq, void *dev)
  55 {
  56         struct hns3_enet_tqp_vector *tqp_vector = dev;
  57
  58         napi_schedule(&tqp_vector->napi);
  59
  60         return IRQ_HANDLED;
  61 }
  62
  63 static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
  64 {
  65         struct hns3_enet_tqp_vector *tqp_vectors;
  66         unsigned int i;
  67
  68         for (i = 0; i < priv->vector_num; i++) {
  69                 tqp_vectors = &priv->tqp_vector[i];
  70
  71                 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
  72                         continue;
  73
  74                 /* release the irq resource */
  75                 free_irq(tqp_vectors->vector_irq, tqp_vectors);
  76                 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
  77         }
  78 }
  79
  80 static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
  81 {
  82         struct hns3_enet_tqp_vector *tqp_vectors;
  83         int txrx_int_idx = 0;
  84         int rx_int_idx = 0;
  85         int tx_int_idx = 0;
  86         unsigned int i;
  87         int ret;
  88
  89         for (i = 0; i < priv->vector_num; i++) {
  90                 tqp_vectors = &priv->tqp_vector[i];
  91
  92                 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
  93                         continue;
  94
  95                 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
  96                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
  97                                  "%s-%s-%d", priv->netdev->name, "TxRx",
  98                                  txrx_int_idx++);
  99                         txrx_int_idx++;
 100                 } else if (tqp_vectors->rx_group.ring) {
 101                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
 102                                  "%s-%s-%d", priv->netdev->name, "Rx",
 103                                  rx_int_idx++);
 104                 } else if (tqp_vectors->tx_group.ring) {
 105                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
 106                                  "%s-%s-%d", priv->netdev->name, "Tx",
 107                                  tx_int_idx++);
 108                 } else {
 109                         /* Skip this unused q_vector */
 110                         continue;
 111                 }
 112
 113                 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
 114
 115                 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
 116                                   tqp_vectors->name,
 117                                        tqp_vectors);
 118                 if (ret) {
 119                         netdev_err(priv->netdev, "request irq(%d) fail\n",
 120                                    tqp_vectors->vector_irq);
 121                         return ret;
 122                 }
 123
 124                 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
 125         }
 126
 127         return 0;
 128 }
 129
 130 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
 131                                  u32 mask_en)
 132 {
 133         writel(mask_en, tqp_vector->mask_addr);
 134 }
 135
 136 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
 137 {
 138         napi_enable(&tqp_vector->napi);
 139
 140         /* enable vector */
 141         hns3_mask_vector_irq(tqp_vector, 1);
 142 }
 143
 144 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
 145 {
 146         /* disable vector */
 147         hns3_mask_vector_irq(tqp_vector, 0);
 148
 149         disable_irq(tqp_vector->vector_irq);
 150         napi_disable(&tqp_vector->napi);
 151 }
 152
 153 static void hns3_set_vector_coalesc_gl(struct hns3_enet_tqp_vector *tqp_vector,
 154                                        u32 gl_value)
 155 {
 156         /* this defines the configuration for GL (Interrupt Gap Limiter)
 157          * GL defines inter interrupt gap.
 158          * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
 159          */
 160         writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
 161         writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
 162         writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL2_OFFSET);
 163 }
 164
 165 static void hns3_set_vector_coalesc_rl(struct hns3_enet_tqp_vector *tqp_vector,
 166                                        u32 rl_value)
 167 {
 168         /* this defines the configuration for RL (Interrupt Rate Limiter).
 169          * Rl defines rate of interrupts i.e. number of interrupts-per-second
 170          * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
 171          */
 172         writel(rl_value, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
 173 }
 174
 175 static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector)
 176 {
 177         /* initialize the configuration for interrupt coalescing.
 178          * 1. GL (Interrupt Gap Limiter)
 179          * 2. RL (Interrupt Rate Limiter)
 180          */
 181
 182         /* Default :enable interrupt coalesce */
 183         tqp_vector->rx_group.int_gl = HNS3_INT_GL_50K;
 184         tqp_vector->tx_group.int_gl = HNS3_INT_GL_50K;
 185         hns3_set_vector_coalesc_gl(tqp_vector, HNS3_INT_GL_50K);
 186         /* for now we are disabling Interrupt RL - we
 187          * will re-enable later
 188          */
 189         hns3_set_vector_coalesc_rl(tqp_vector, 0);
 190         tqp_vector->rx_group.flow_level = HNS3_FLOW_LOW;
 191         tqp_vector->tx_group.flow_level = HNS3_FLOW_LOW;
 192 }
 193
 194 static int hns3_nic_net_up(struct net_device *netdev)
 195 {
 196         struct hns3_nic_priv *priv = netdev_priv(netdev);
 197         struct hnae3_handle *h = priv->ae_handle;
 198         int i, j;
 199         int ret;
 200
 201         /* get irq resource for all vectors */
 202         ret = hns3_nic_init_irq(priv);
 203         if (ret) {
 204                 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
 205                 return ret;
 206         }
 207
 208         /* enable the vectors */
 209         for (i = 0; i < priv->vector_num; i++)
 210                 hns3_vector_enable(&priv->tqp_vector[i]);
 211
 212         /* start the ae_dev */
 213         ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
 214         if (ret)
 215                 goto out_start_err;
 216
 217         return 0;
 218
 219 out_start_err:
 220         for (j = i - 1; j >= 0; j--)
 221                 hns3_vector_disable(&priv->tqp_vector[j]);
 222
 223         hns3_nic_uninit_irq(priv);
 224
 225         return ret;
 226 }
 227
 228 static int hns3_nic_net_open(struct net_device *netdev)
 229 {
 230         struct hns3_nic_priv *priv = netdev_priv(netdev);
 231         struct hnae3_handle *h = priv->ae_handle;
 232         int ret;
 233
 234         netif_carrier_off(netdev);
 235
 236         ret = netif_set_real_num_tx_queues(netdev, h->kinfo.num_tqps);
 237         if (ret) {
 238                 netdev_err(netdev,
 239                            "netif_set_real_num_tx_queues fail, ret=%d!\n",
 240                            ret);
 241                 return ret;
 242         }
 243
 244         ret = netif_set_real_num_rx_queues(netdev, h->kinfo.num_tqps);
 245         if (ret) {
 246                 netdev_err(netdev,
 247                            "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
 248                 return ret;
 249         }
 250
 251         ret = hns3_nic_net_up(netdev);
 252         if (ret) {
 253                 netdev_err(netdev,
 254                            "hns net up fail, ret=%d!\n", ret);
 255                 return ret;
 256         }
 257
 258         return 0;
 259 }
 260
 261 static void hns3_nic_net_down(struct net_device *netdev)
 262 {
 263         struct hns3_nic_priv *priv = netdev_priv(netdev);
 264         const struct hnae3_ae_ops *ops;
 265         int i;
 266
 267         /* stop ae_dev */
 268         ops = priv->ae_handle->ae_algo->ops;
 269         if (ops->stop)
 270                 ops->stop(priv->ae_handle);
 271
 272         /* disable vectors */
 273         for (i = 0; i < priv->vector_num; i++)
 274                 hns3_vector_disable(&priv->tqp_vector[i]);
 275
 276         /* free irq resources */
 277         hns3_nic_uninit_irq(priv);
 278 }
 279
 280 static int hns3_nic_net_stop(struct net_device *netdev)
 281 {
 282         netif_tx_stop_all_queues(netdev);
 283         netif_carrier_off(netdev);
 284
 285         hns3_nic_net_down(netdev);
 286
 287         return 0;
 288 }
 289
 290 void hns3_set_multicast_list(struct net_device *netdev)
 291 {
 292         struct hns3_nic_priv *priv = netdev_priv(netdev);
 293         struct hnae3_handle *h = priv->ae_handle;
 294         struct netdev_hw_addr *ha = NULL;
 295
 296         if (h->ae_algo->ops->set_mc_addr) {
 297                 netdev_for_each_mc_addr(ha, netdev)
 298                         if (h->ae_algo->ops->set_mc_addr(h, ha->addr))
 299                                 netdev_err(netdev, "set multicast fail\n");
 300         }
 301 }
 302
 303 static int hns3_nic_uc_sync(struct net_device *netdev,
 304                             const unsigned char *addr)
 305 {
 306         struct hns3_nic_priv *priv = netdev_priv(netdev);
 307         struct hnae3_handle *h = priv->ae_handle;
 308
 309         if (h->ae_algo->ops->add_uc_addr)
 310                 return h->ae_algo->ops->add_uc_addr(h, addr);
 311
 312         return 0;
 313 }
 314
 315 static int hns3_nic_uc_unsync(struct net_device *netdev,
 316                               const unsigned char *addr)
 317 {
 318         struct hns3_nic_priv *priv = netdev_priv(netdev);
 319         struct hnae3_handle *h = priv->ae_handle;
 320
 321         if (h->ae_algo->ops->rm_uc_addr)
 322                 return h->ae_algo->ops->rm_uc_addr(h, addr);
 323
 324         return 0;
 325 }
 326
 327 static int hns3_nic_mc_sync(struct net_device *netdev,
 328                             const unsigned char *addr)
 329 {
 330         struct hns3_nic_priv *priv = netdev_priv(netdev);
 331         struct hnae3_handle *h = priv->ae_handle;
 332
 333         if (h->ae_algo->ops->add_mc_addr)
 334                 return h->ae_algo->ops->add_mc_addr(h, addr);
 335
 336         return 0;
 337 }
 338
 339 static int hns3_nic_mc_unsync(struct net_device *netdev,
 340                               const unsigned char *addr)
 341 {
 342         struct hns3_nic_priv *priv = netdev_priv(netdev);
 343         struct hnae3_handle *h = priv->ae_handle;
 344
 345         if (h->ae_algo->ops->rm_mc_addr)
 346                 return h->ae_algo->ops->rm_mc_addr(h, addr);
 347
 348         return 0;
 349 }
 350
 351 void hns3_nic_set_rx_mode(struct net_device *netdev)
 352 {
 353         struct hns3_nic_priv *priv = netdev_priv(netdev);
 354         struct hnae3_handle *h = priv->ae_handle;
 355
 356         if (h->ae_algo->ops->set_promisc_mode) {
 357                 if (netdev->flags & IFF_PROMISC)
 358                         h->ae_algo->ops->set_promisc_mode(h, 1);
 359                 else
 360                         h->ae_algo->ops->set_promisc_mode(h, 0);
 361         }
 362         if (__dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync))
 363                 netdev_err(netdev, "sync uc address fail\n");
 364         if (netdev->flags & IFF_MULTICAST)
 365                 if (__dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync))
 366                         netdev_err(netdev, "sync mc address fail\n");
 367 }
 368
 369 static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
 370                         u16 *mss, u32 *type_cs_vlan_tso)
 371 {
 372         u32 l4_offset, hdr_len;
 373         union l3_hdr_info l3;
 374         union l4_hdr_info l4;
 375         u32 l4_paylen;
 376         int ret;
 377
 378         if (!skb_is_gso(skb))
 379                 return 0;
 380
 381         ret = skb_cow_head(skb, 0);
 382         if (ret)
 383                 return ret;
 384
 385         l3.hdr = skb_network_header(skb);
 386         l4.hdr = skb_transport_header(skb);
 387
 388         /* Software should clear the IPv4's checksum field when tso is
 389          * needed.
 390          */
 391         if (l3.v4->version == 4)
 392                 l3.v4->check = 0;
 393
 394         /* tunnel packet.*/
 395         if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
 396                                          SKB_GSO_GRE_CSUM |
 397                                          SKB_GSO_UDP_TUNNEL |
 398                                          SKB_GSO_UDP_TUNNEL_CSUM)) {
 399                 if ((!(skb_shinfo(skb)->gso_type &
 400                     SKB_GSO_PARTIAL)) &&
 401                     (skb_shinfo(skb)->gso_type &
 402                     SKB_GSO_UDP_TUNNEL_CSUM)) {
 403                         /* Software should clear the udp's checksum
 404                          * field when tso is needed.
 405                          */
 406                         l4.udp->check = 0;
 407                 }
 408                 /* reset l3&l4 pointers from outer to inner headers */
 409                 l3.hdr = skb_inner_network_header(skb);
 410                 l4.hdr = skb_inner_transport_header(skb);
 411
 412                 /* Software should clear the IPv4's checksum field when
 413                  * tso is needed.
 414                  */
 415                 if (l3.v4->version == 4)
 416                         l3.v4->check = 0;
 417         }
 418
 419         /* normal or tunnel packet*/
 420         l4_offset = l4.hdr - skb->data;
 421         hdr_len = (l4.tcp->doff * 4) + l4_offset;
 422
 423         /* remove payload length from inner pseudo checksum when tso*/
 424         l4_paylen = skb->len - l4_offset;
 425         csum_replace_by_diff(&l4.tcp->check,
 426                              (__force __wsum)htonl(l4_paylen));
 427
 428         /* find the txbd field values */
 429         *paylen = skb->len - hdr_len;
 430         hnae_set_bit(*type_cs_vlan_tso,
 431                      HNS3_TXD_TSO_B, 1);
 432
 433         /* get MSS for TSO */
 434         *mss = skb_shinfo(skb)->gso_size;
 435
 436         return 0;
 437 }
 438
 439 static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
 440                                 u8 *il4_proto)
 441 {
 442         union {
 443                 struct iphdr *v4;
 444                 struct ipv6hdr *v6;
 445                 unsigned char *hdr;
 446         } l3;
 447         unsigned char *l4_hdr;
 448         unsigned char *exthdr;
 449         u8 l4_proto_tmp;
 450         __be16 frag_off;
 451
 452         /* find outer header point */
 453         l3.hdr = skb_network_header(skb);
 454         l4_hdr = skb_inner_transport_header(skb);
 455
 456         if (skb->protocol == htons(ETH_P_IPV6)) {
 457                 exthdr = l3.hdr + sizeof(*l3.v6);
 458                 l4_proto_tmp = l3.v6->nexthdr;
 459                 if (l4_hdr != exthdr)
 460                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 461                                          &l4_proto_tmp, &frag_off);
 462         } else if (skb->protocol == htons(ETH_P_IP)) {
 463                 l4_proto_tmp = l3.v4->protocol;
 464         } else {
 465                 return -EINVAL;
 466         }
 467
 468         *ol4_proto = l4_proto_tmp;
 469
 470         /* tunnel packet */
 471         if (!skb->encapsulation) {
 472                 *il4_proto = 0;
 473                 return 0;
 474         }
 475
 476         /* find inner header point */
 477         l3.hdr = skb_inner_network_header(skb);
 478         l4_hdr = skb_inner_transport_header(skb);
 479
 480         if (l3.v6->version == 6) {
 481                 exthdr = l3.hdr + sizeof(*l3.v6);
 482                 l4_proto_tmp = l3.v6->nexthdr;
 483                 if (l4_hdr != exthdr)
 484                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 485                                          &l4_proto_tmp, &frag_off);
 486         } else if (l3.v4->version == 4) {
 487                 l4_proto_tmp = l3.v4->protocol;
 488         }
 489
 490         *il4_proto = l4_proto_tmp;
 491
 492         return 0;
 493 }
 494
 495 static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
 496                                 u8 il4_proto, u32 *type_cs_vlan_tso,
 497                                 u32 *ol_type_vlan_len_msec)
 498 {
 499         union {
 500                 struct iphdr *v4;
 501                 struct ipv6hdr *v6;
 502                 unsigned char *hdr;
 503         } l3;
 504         union {
 505                 struct tcphdr *tcp;
 506                 struct udphdr *udp;
 507                 struct gre_base_hdr *gre;
 508                 unsigned char *hdr;
 509         } l4;
 510         unsigned char *l2_hdr;
 511         u8 l4_proto = ol4_proto;
 512         u32 ol2_len;
 513         u32 ol3_len;
 514         u32 ol4_len;
 515         u32 l2_len;
 516         u32 l3_len;
 517
 518         l3.hdr = skb_network_header(skb);
 519         l4.hdr = skb_transport_header(skb);
 520
 521         /* compute L2 header size for normal packet, defined in 2 Bytes */
 522         l2_len = l3.hdr - skb->data;
 523         hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
 524                        HNS3_TXD_L2LEN_S, l2_len >> 1);
 525
 526         /* tunnel packet*/
 527         if (skb->encapsulation) {
 528                 /* compute OL2 header size, defined in 2 Bytes */
 529                 ol2_len = l2_len;
 530                 hnae_set_field(*ol_type_vlan_len_msec,
 531                                HNS3_TXD_L2LEN_M,
 532                                HNS3_TXD_L2LEN_S, ol2_len >> 1);
 533
 534                 /* compute OL3 header size, defined in 4 Bytes */
 535                 ol3_len = l4.hdr - l3.hdr;
 536                 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
 537                                HNS3_TXD_L3LEN_S, ol3_len >> 2);
 538
 539                 /* MAC in UDP, MAC in GRE (0x6558)*/
 540                 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
 541                         /* switch MAC header ptr from outer to inner header.*/
 542                         l2_hdr = skb_inner_mac_header(skb);
 543
 544                         /* compute OL4 header size, defined in 4 Bytes. */
 545                         ol4_len = l2_hdr - l4.hdr;
 546                         hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_M,
 547                                        HNS3_TXD_L4LEN_S, ol4_len >> 2);
 548
 549                         /* switch IP header ptr from outer to inner header */
 550                         l3.hdr = skb_inner_network_header(skb);
 551
 552                         /* compute inner l2 header size, defined in 2 Bytes. */
 553                         l2_len = l3.hdr - l2_hdr;
 554                         hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
 555                                        HNS3_TXD_L2LEN_S, l2_len >> 1);
 556                 } else {
 557                         /* skb packet types not supported by hardware,
 558                          * txbd len fild doesn't be filled.
 559                          */
 560                         return;
 561                 }
 562
 563                 /* switch L4 header pointer from outer to inner */
 564                 l4.hdr = skb_inner_transport_header(skb);
 565
 566                 l4_proto = il4_proto;
 567         }
 568
 569         /* compute inner(/normal) L3 header size, defined in 4 Bytes */
 570         l3_len = l4.hdr - l3.hdr;
 571         hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
 572                        HNS3_TXD_L3LEN_S, l3_len >> 2);
 573
 574         /* compute inner(/normal) L4 header size, defined in 4 Bytes */
 575         switch (l4_proto) {
 576         case IPPROTO_TCP:
 577                 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 578                                HNS3_TXD_L4LEN_S, l4.tcp->doff);
 579                 break;
 580         case IPPROTO_SCTP:
 581                 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 582                                HNS3_TXD_L4LEN_S, (sizeof(struct sctphdr) >> 2));
 583                 break;
 584         case IPPROTO_UDP:
 585                 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 586                                HNS3_TXD_L4LEN_S, (sizeof(struct udphdr) >> 2));
 587                 break;
 588         default:
 589                 /* skb packet types not supported by hardware,
 590                  * txbd len fild doesn't be filled.
 591                  */
 592                 return;
 593         }
 594 }
 595
 596 static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
 597                                    u8 il4_proto, u32 *type_cs_vlan_tso,
 598                                    u32 *ol_type_vlan_len_msec)
 599 {
 600         union {
 601                 struct iphdr *v4;
 602                 struct ipv6hdr *v6;
 603                 unsigned char *hdr;
 604         } l3;
 605         u32 l4_proto = ol4_proto;
 606
 607         l3.hdr = skb_network_header(skb);
 608
 609         /* define OL3 type and tunnel type(OL4).*/
 610         if (skb->encapsulation) {
 611                 /* define outer network header type.*/
 612                 if (skb->protocol == htons(ETH_P_IP)) {
 613                         if (skb_is_gso(skb))
 614                                 hnae_set_field(*ol_type_vlan_len_msec,
 615                                                HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
 616                                                HNS3_OL3T_IPV4_CSUM);
 617                         else
 618                                 hnae_set_field(*ol_type_vlan_len_msec,
 619                                                HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
 620                                                HNS3_OL3T_IPV4_NO_CSUM);
 621
 622                 } else if (skb->protocol == htons(ETH_P_IPV6)) {
 623                         hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
 624                                        HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
 625                 }
 626
 627                 /* define tunnel type(OL4).*/
 628                 switch (l4_proto) {
 629                 case IPPROTO_UDP:
 630                         hnae_set_field(*ol_type_vlan_len_msec,
 631                                        HNS3_TXD_TUNTYPE_M,
 632                                        HNS3_TXD_TUNTYPE_S,
 633                                        HNS3_TUN_MAC_IN_UDP);
 634                         break;
 635                 case IPPROTO_GRE:
 636                         hnae_set_field(*ol_type_vlan_len_msec,
 637                                        HNS3_TXD_TUNTYPE_M,
 638                                        HNS3_TXD_TUNTYPE_S,
 639                                        HNS3_TUN_NVGRE);
 640                         break;
 641                 default:
 642                         /* drop the skb tunnel packet if hardware don't support,
 643                          * because hardware can't calculate csum when TSO.
 644                          */
 645                         if (skb_is_gso(skb))
 646                                 return -EDOM;
 647
 648                         /* the stack computes the IP header already,
 649                          * driver calculate l4 checksum when not TSO.
 650                          */
 651                         skb_checksum_help(skb);
 652                         return 0;
 653                 }
 654
 655                 l3.hdr = skb_inner_network_header(skb);
 656                 l4_proto = il4_proto;
 657         }
 658
 659         if (l3.v4->version == 4) {
 660                 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
 661                                HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
 662
 663                 /* the stack computes the IP header already, the only time we
 664                  * need the hardware to recompute it is in the case of TSO.
 665                  */
 666                 if (skb_is_gso(skb))
 667                         hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
 668
 669                 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 670         } else if (l3.v6->version == 6) {
 671                 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
 672                                HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
 673                 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 674         }
 675
 676         switch (l4_proto) {
 677         case IPPROTO_TCP:
 678                 hnae_set_field(*type_cs_vlan_tso,
 679                                HNS3_TXD_L4T_M,
 680                                HNS3_TXD_L4T_S,
 681                                HNS3_L4T_TCP);
 682                 break;
 683         case IPPROTO_UDP:
 684                 hnae_set_field(*type_cs_vlan_tso,
 685                                HNS3_TXD_L4T_M,
 686                                HNS3_TXD_L4T_S,
 687                                HNS3_L4T_UDP);
 688                 break;
 689         case IPPROTO_SCTP:
 690                 hnae_set_field(*type_cs_vlan_tso,
 691                                HNS3_TXD_L4T_M,
 692                                HNS3_TXD_L4T_S,
 693                                HNS3_L4T_SCTP);
 694                 break;
 695         default:
 696                 /* drop the skb tunnel packet if hardware don't support,
 697                  * because hardware can't calculate csum when TSO.
 698                  */
 699                 if (skb_is_gso(skb))
 700                         return -EDOM;
 701
 702                 /* the stack computes the IP header already,
 703                  * driver calculate l4 checksum when not TSO.
 704                  */
 705                 skb_checksum_help(skb);
 706                 return 0;
 707         }
 708
 709         return 0;
 710 }
 711
 712 static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
 713 {
 714         /* Config bd buffer end */
 715         hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
 716                        HNS3_TXD_BDTYPE_M, 0);
 717         hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
 718         hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
 719         hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 1);
 720 }
 721
 722 static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
 723                           int size, dma_addr_t dma, int frag_end,
 724                           enum hns_desc_type type)
 725 {
 726         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
 727         struct hns3_desc *desc = &ring->desc[ring->next_to_use];
 728         u32 ol_type_vlan_len_msec = 0;
 729         u16 bdtp_fe_sc_vld_ra_ri = 0;
 730         u32 type_cs_vlan_tso = 0;
 731         struct sk_buff *skb;
 732         u32 paylen = 0;
 733         u16 mss = 0;
 734         __be16 protocol;
 735         u8 ol4_proto;
 736         u8 il4_proto;
 737         int ret;
 738
 739         /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
 740         desc_cb->priv = priv;
 741         desc_cb->length = size;
 742         desc_cb->dma = dma;
 743         desc_cb->type = type;
 744
 745         /* now, fill the descriptor */
 746         desc->addr = cpu_to_le64(dma);
 747         desc->tx.send_size = cpu_to_le16((u16)size);
 748         hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri, frag_end);
 749         desc->tx.bdtp_fe_sc_vld_ra_ri = cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
 750
 751         if (type == DESC_TYPE_SKB) {
 752                 skb = (struct sk_buff *)priv;
 753                 paylen = cpu_to_le16(skb->len);
 754
 755                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
 756                         skb_reset_mac_len(skb);
 757                         protocol = skb->protocol;
 758
 759                         /* vlan packet*/
 760                         if (protocol == htons(ETH_P_8021Q)) {
 761                                 protocol = vlan_get_protocol(skb);
 762                                 skb->protocol = protocol;
 763                         }
 764                         ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
 765                         if (ret)
 766                                 return ret;
 767                         hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
 768                                             &type_cs_vlan_tso,
 769                                             &ol_type_vlan_len_msec);
 770                         ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
 771                                                       &type_cs_vlan_tso,
 772                                                       &ol_type_vlan_len_msec);
 773                         if (ret)
 774                                 return ret;
 775
 776                         ret = hns3_set_tso(skb, &paylen, &mss,
 777                                            &type_cs_vlan_tso);
 778                         if (ret)
 779                                 return ret;
 780                 }
 781
 782                 /* Set txbd */
 783                 desc->tx.ol_type_vlan_len_msec =
 784                         cpu_to_le32(ol_type_vlan_len_msec);
 785                 desc->tx.type_cs_vlan_tso_len =
 786                         cpu_to_le32(type_cs_vlan_tso);
 787                 desc->tx.paylen = cpu_to_le16(paylen);
 788                 desc->tx.mss = cpu_to_le16(mss);
 789         }
 790
 791         /* move ring pointer to next.*/
 792         ring_ptr_move_fw(ring, next_to_use);
 793
 794         return 0;
 795 }
 796
 797 static int hns3_fill_desc_tso(struct hns3_enet_ring *ring, void *priv,
 798                               int size, dma_addr_t dma, int frag_end,
 799                               enum hns_desc_type type)
 800 {
 801         unsigned int frag_buf_num;
 802         unsigned int k;
 803         int sizeoflast;
 804         int ret;
 805
 806         frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
 807         sizeoflast = size % HNS3_MAX_BD_SIZE;
 808         sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
 809
 810         /* When the frag size is bigger than hardware, split this frag */
 811         for (k = 0; k < frag_buf_num; k++) {
 812                 ret = hns3_fill_desc(ring, priv,
 813                                      (k == frag_buf_num - 1) ?
 814                                 sizeoflast : HNS3_MAX_BD_SIZE,
 815                                 dma + HNS3_MAX_BD_SIZE * k,
 816                                 frag_end && (k == frag_buf_num - 1) ? 1 : 0,
 817                                 (type == DESC_TYPE_SKB && !k) ?
 818                                         DESC_TYPE_SKB : DESC_TYPE_PAGE);
 819                 if (ret)
 820                         return ret;
 821         }
 822
 823         return 0;
 824 }
 825
 826 static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
 827                                    struct hns3_enet_ring *ring)
 828 {
 829         struct sk_buff *skb = *out_skb;
 830         struct skb_frag_struct *frag;
 831         int bdnum_for_frag;
 832         int frag_num;
 833         int buf_num;
 834         int size;
 835         int i;
 836
 837         size = skb_headlen(skb);
 838         buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
 839
 840         frag_num = skb_shinfo(skb)->nr_frags;
 841         for (i = 0; i < frag_num; i++) {
 842                 frag = &skb_shinfo(skb)->frags[i];
 843                 size = skb_frag_size(frag);
 844                 bdnum_for_frag =
 845                         (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
 846                 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
 847                         return -ENOMEM;
 848
 849                 buf_num += bdnum_for_frag;
 850         }
 851
 852         if (buf_num > ring_space(ring))
 853                 return -EBUSY;
 854
 855         *bnum = buf_num;
 856         return 0;
 857 }
 858
 859 static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
 860                                   struct hns3_enet_ring *ring)
 861 {
 862         struct sk_buff *skb = *out_skb;
 863         int buf_num;
 864
 865         /* No. of segments (plus a header) */
 866         buf_num = skb_shinfo(skb)->nr_frags + 1;
 867
 868         if (buf_num > ring_space(ring))
 869                 return -EBUSY;
 870
 871         *bnum = buf_num;
 872
 873         return 0;
 874 }
 875
 876 static void hns_nic_dma_unmap(struct hns3_enet_ring *ring, int next_to_use_orig)
 877 {
 878         struct device *dev = ring_to_dev(ring);
 879         unsigned int i;
 880
 881         for (i = 0; i < ring->desc_num; i++) {
 882                 /* check if this is where we started */
 883                 if (ring->next_to_use == next_to_use_orig)
 884                         break;
 885
 886                 /* unmap the descriptor dma address */
 887                 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
 888                         dma_unmap_single(dev,
 889                                          ring->desc_cb[ring->next_to_use].dma,
 890                                         ring->desc_cb[ring->next_to_use].length,
 891                                         DMA_TO_DEVICE);
 892                 else
 893                         dma_unmap_page(dev,
 894                                        ring->desc_cb[ring->next_to_use].dma,
 895                                        ring->desc_cb[ring->next_to_use].length,
 896                                        DMA_TO_DEVICE);
 897
 898                 /* rollback one */
 899                 ring_ptr_move_bw(ring, next_to_use);
 900         }
 901 }
 902
 903 static netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb,
 904                                      struct net_device *netdev)
 905 {
 906         struct hns3_nic_priv *priv = netdev_priv(netdev);
 907         struct hns3_nic_ring_data *ring_data =
 908                 &tx_ring_data(priv, skb->queue_mapping);
 909         struct hns3_enet_ring *ring = ring_data->ring;
 910         struct device *dev = priv->dev;
 911         struct netdev_queue *dev_queue;
 912         struct skb_frag_struct *frag;
 913         int next_to_use_head;
 914         int next_to_use_frag;
 915         dma_addr_t dma;
 916         int buf_num;
 917         int seg_num;
 918         int size;
 919         int ret;
 920         int i;
 921
 922         /* Prefetch the data used later */
 923         prefetch(skb->data);
 924
 925         switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
 926         case -EBUSY:
 927                 u64_stats_update_begin(&ring->syncp);
 928                 ring->stats.tx_busy++;
 929                 u64_stats_update_end(&ring->syncp);
 930
 931                 goto out_net_tx_busy;
 932         case -ENOMEM:
 933                 u64_stats_update_begin(&ring->syncp);
 934                 ring->stats.sw_err_cnt++;
 935                 u64_stats_update_end(&ring->syncp);
 936                 netdev_err(netdev, "no memory to xmit!\n");
 937
 938                 goto out_err_tx_ok;
 939         default:
 940                 break;
 941         }
 942
 943         /* No. of segments (plus a header) */
 944         seg_num = skb_shinfo(skb)->nr_frags + 1;
 945         /* Fill the first part */
 946         size = skb_headlen(skb);
 947
 948         next_to_use_head = ring->next_to_use;
 949
 950         dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
 951         if (dma_mapping_error(dev, dma)) {
 952                 netdev_err(netdev, "TX head DMA map failed\n");
 953                 ring->stats.sw_err_cnt++;
 954                 goto out_err_tx_ok;
 955         }
 956
 957         ret = priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0,
 958                            DESC_TYPE_SKB);
 959         if (ret)
 960                 goto head_dma_map_err;
 961
 962         next_to_use_frag = ring->next_to_use;
 963         /* Fill the fragments */
 964         for (i = 1; i < seg_num; i++) {
 965                 frag = &skb_shinfo(skb)->frags[i - 1];
 966                 size = skb_frag_size(frag);
 967                 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
 968                 if (dma_mapping_error(dev, dma)) {
 969                         netdev_err(netdev, "TX frag(%d) DMA map failed\n", i);
 970                         ring->stats.sw_err_cnt++;
 971                         goto frag_dma_map_err;
 972                 }
 973                 ret = priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma,
 974                                     seg_num - 1 == i ? 1 : 0,
 975                                     DESC_TYPE_PAGE);
 976
 977                 if (ret)
 978                         goto frag_dma_map_err;
 979         }
 980
 981         /* Complete translate all packets */
 982         dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
 983         netdev_tx_sent_queue(dev_queue, skb->len);
 984
 985         wmb(); /* Commit all data before submit */
 986
 987         hnae_queue_xmit(ring->tqp, buf_num);
 988
 989         return NETDEV_TX_OK;
 990
 991 frag_dma_map_err:
 992         hns_nic_dma_unmap(ring, next_to_use_frag);
 993
 994 head_dma_map_err:
 995         hns_nic_dma_unmap(ring, next_to_use_head);
 996
 997 out_err_tx_ok:
 998         dev_kfree_skb_any(skb);
 999         return NETDEV_TX_OK;
1000
1001 out_net_tx_busy:
1002         netif_stop_subqueue(netdev, ring_data->queue_index);
1003         smp_mb(); /* Commit all data before submit */
1004
1005         return NETDEV_TX_BUSY;
1006 }
1007
1008 static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1009 {
1010         struct hns3_nic_priv *priv = netdev_priv(netdev);
1011         struct hnae3_handle *h = priv->ae_handle;
1012         struct sockaddr *mac_addr = p;
1013         int ret;
1014
1015         if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1016                 return -EADDRNOTAVAIL;
1017
1018         ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data);
1019         if (ret) {
1020                 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1021                 return ret;
1022         }
1023
1024         ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1025
1026         return 0;
1027 }
1028
1029 static int hns3_nic_set_features(struct net_device *netdev,
1030                                  netdev_features_t features)
1031 {
1032         struct hns3_nic_priv *priv = netdev_priv(netdev);
1033
1034         if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
1035                 priv->ops.fill_desc = hns3_fill_desc_tso;
1036                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1037         } else {
1038                 priv->ops.fill_desc = hns3_fill_desc;
1039                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1040         }
1041
1042         netdev->features = features;
1043         return 0;
1044 }
1045
1046 static void
1047 hns3_nic_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
1048 {
1049         struct hns3_nic_priv *priv = netdev_priv(netdev);
1050         int queue_num = priv->ae_handle->kinfo.num_tqps;
1051         struct hns3_enet_ring *ring;
1052         unsigned int start;
1053         unsigned int idx;
1054         u64 tx_bytes = 0;
1055         u64 rx_bytes = 0;
1056         u64 tx_pkts = 0;
1057         u64 rx_pkts = 0;
1058
1059         for (idx = 0; idx < queue_num; idx++) {
1060                 /* fetch the tx stats */
1061                 ring = priv->ring_data[idx].ring;
1062                 do {
1063                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1064                         tx_bytes += ring->stats.tx_bytes;
1065                         tx_pkts += ring->stats.tx_pkts;
1066                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1067
1068                 /* fetch the rx stats */
1069                 ring = priv->ring_data[idx + queue_num].ring;
1070                 do {
1071                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1072                         rx_bytes += ring->stats.rx_bytes;
1073                         rx_pkts += ring->stats.rx_pkts;
1074                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1075         }
1076
1077         stats->tx_bytes = tx_bytes;
1078         stats->tx_packets = tx_pkts;
1079         stats->rx_bytes = rx_bytes;
1080         stats->rx_packets = rx_pkts;
1081
1082         stats->rx_errors = netdev->stats.rx_errors;
1083         stats->multicast = netdev->stats.multicast;
1084         stats->rx_length_errors = netdev->stats.rx_length_errors;
1085         stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1086         stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1087
1088         stats->tx_errors = netdev->stats.tx_errors;
1089         stats->rx_dropped = netdev->stats.rx_dropped;
1090         stats->tx_dropped = netdev->stats.tx_dropped;
1091         stats->collisions = netdev->stats.collisions;
1092         stats->rx_over_errors = netdev->stats.rx_over_errors;
1093         stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1094         stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1095         stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1096         stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1097         stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1098         stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1099         stats->tx_window_errors = netdev->stats.tx_window_errors;
1100         stats->rx_compressed = netdev->stats.rx_compressed;
1101         stats->tx_compressed = netdev->stats.tx_compressed;
1102 }
1103
1104 static void hns3_add_tunnel_port(struct net_device *netdev, u16 port,
1105                                  enum hns3_udp_tnl_type type)
1106 {
1107         struct hns3_nic_priv *priv = netdev_priv(netdev);
1108         struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1109         struct hnae3_handle *h = priv->ae_handle;
1110
1111         if (udp_tnl->used && udp_tnl->dst_port == port) {
1112                 udp_tnl->used++;
1113                 return;
1114         }
1115
1116         if (udp_tnl->used) {
1117                 netdev_warn(netdev,
1118                             "UDP tunnel [%d], port [%d] offload\n", type, port);
1119                 return;
1120         }
1121
1122         udp_tnl->dst_port = port;
1123         udp_tnl->used = 1;
1124         /* TBD send command to hardware to add port */
1125         if (h->ae_algo->ops->add_tunnel_udp)
1126                 h->ae_algo->ops->add_tunnel_udp(h, port);
1127 }
1128
1129 static void hns3_del_tunnel_port(struct net_device *netdev, u16 port,
1130                                  enum hns3_udp_tnl_type type)
1131 {
1132         struct hns3_nic_priv *priv = netdev_priv(netdev);
1133         struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1134         struct hnae3_handle *h = priv->ae_handle;
1135
1136         if (!udp_tnl->used || udp_tnl->dst_port != port) {
1137                 netdev_warn(netdev,
1138                             "Invalid UDP tunnel port %d\n", port);
1139                 return;
1140         }
1141
1142         udp_tnl->used--;
1143         if (udp_tnl->used)
1144                 return;
1145
1146         udp_tnl->dst_port = 0;
1147         /* TBD send command to hardware to del port  */
1148         if (h->ae_algo->ops->del_tunnel_udp)
1149                 h->ae_algo->ops->del_tunnel_udp(h, port);
1150 }
1151
1152 /* hns3_nic_udp_tunnel_add - Get notifiacetion about UDP tunnel ports
1153  * @netdev: This physical ports's netdev
1154  * @ti: Tunnel information
1155  */
1156 static void hns3_nic_udp_tunnel_add(struct net_device *netdev,
1157                                     struct udp_tunnel_info *ti)
1158 {
1159         u16 port_n = ntohs(ti->port);
1160
1161         switch (ti->type) {
1162         case UDP_TUNNEL_TYPE_VXLAN:
1163                 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1164                 break;
1165         case UDP_TUNNEL_TYPE_GENEVE:
1166                 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1167                 break;
1168         default:
1169                 netdev_err(netdev, "unsupported tunnel type %d\n", ti->type);
1170                 break;
1171         }
1172 }
1173
1174 static void hns3_nic_udp_tunnel_del(struct net_device *netdev,
1175                                     struct udp_tunnel_info *ti)
1176 {
1177         u16 port_n = ntohs(ti->port);
1178
1179         switch (ti->type) {
1180         case UDP_TUNNEL_TYPE_VXLAN:
1181                 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1182                 break;
1183         case UDP_TUNNEL_TYPE_GENEVE:
1184                 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1185                 break;
1186         default:
1187                 break;
1188         }
1189 }
1190
1191 static int hns3_setup_tc(struct net_device *netdev, u8 tc)
1192 {
1193         struct hns3_nic_priv *priv = netdev_priv(netdev);
1194         struct hnae3_handle *h = priv->ae_handle;
1195         struct hnae3_knic_private_info *kinfo = &h->kinfo;
1196         unsigned int i;
1197         int ret;
1198
1199         if (tc > HNAE3_MAX_TC)
1200                 return -EINVAL;
1201
1202         if (kinfo->num_tc == tc)
1203                 return 0;
1204
1205         if (!netdev)
1206                 return -EINVAL;
1207
1208         if (!tc) {
1209                 netdev_reset_tc(netdev);
1210                 return 0;
1211         }
1212
1213         /* Set num_tc for netdev */
1214         ret = netdev_set_num_tc(netdev, tc);
1215         if (ret)
1216                 return ret;
1217
1218         /* Set per TC queues for the VSI */
1219         for (i = 0; i < HNAE3_MAX_TC; i++) {
1220                 if (kinfo->tc_info[i].enable)
1221                         netdev_set_tc_queue(netdev,
1222                                             kinfo->tc_info[i].tc,
1223                                             kinfo->tc_info[i].tqp_count,
1224                                             kinfo->tc_info[i].tqp_offset);
1225         }
1226
1227         return 0;
1228 }
1229
1230 static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
1231                              void *type_data)
1232 {
1233         struct tc_mqprio_qopt *mqprio = type_data;
1234
1235         if (type != TC_SETUP_MQPRIO)
1236                 return -EOPNOTSUPP;
1237
1238         return hns3_setup_tc(dev, mqprio->num_tc);
1239 }
1240
1241 static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1242                                 __be16 proto, u16 vid)
1243 {
1244         struct hns3_nic_priv *priv = netdev_priv(netdev);
1245         struct hnae3_handle *h = priv->ae_handle;
1246         int ret = -EIO;
1247
1248         if (h->ae_algo->ops->set_vlan_filter)
1249                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1250
1251         return ret;
1252 }
1253
1254 static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1255                                  __be16 proto, u16 vid)
1256 {
1257         struct hns3_nic_priv *priv = netdev_priv(netdev);
1258         struct hnae3_handle *h = priv->ae_handle;
1259         int ret = -EIO;
1260
1261         if (h->ae_algo->ops->set_vlan_filter)
1262                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1263
1264         return ret;
1265 }
1266
1267 static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1268                                 u8 qos, __be16 vlan_proto)
1269 {
1270         struct hns3_nic_priv *priv = netdev_priv(netdev);
1271         struct hnae3_handle *h = priv->ae_handle;
1272         int ret = -EIO;
1273
1274         if (h->ae_algo->ops->set_vf_vlan_filter)
1275                 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1276                                                    qos, vlan_proto);
1277
1278         return ret;
1279 }
1280
1281 static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1282 {
1283         struct hns3_nic_priv *priv = netdev_priv(netdev);
1284         struct hnae3_handle *h = priv->ae_handle;
1285         bool if_running = netif_running(netdev);
1286         int ret;
1287
1288         if (!h->ae_algo->ops->set_mtu)
1289                 return -EOPNOTSUPP;
1290
1291         /* if this was called with netdev up then bring netdevice down */
1292         if (if_running) {
1293                 (void)hns3_nic_net_stop(netdev);
1294                 msleep(100);
1295         }
1296
1297         ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1298         if (ret) {
1299                 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1300                            ret);
1301                 return ret;
1302         }
1303
1304         /* if the netdev was running earlier, bring it up again */
1305         if (if_running && hns3_nic_net_open(netdev))
1306                 ret = -EINVAL;
1307
1308         return ret;
1309 }
1310
1311 static const struct net_device_ops hns3_nic_netdev_ops = {
1312         .ndo_open               = hns3_nic_net_open,
1313         .ndo_stop               = hns3_nic_net_stop,
1314         .ndo_start_xmit         = hns3_nic_net_xmit,
1315         .ndo_set_mac_address    = hns3_nic_net_set_mac_address,
1316         .ndo_change_mtu         = hns3_nic_change_mtu,
1317         .ndo_set_features       = hns3_nic_set_features,
1318         .ndo_get_stats64        = hns3_nic_get_stats64,
1319         .ndo_setup_tc           = hns3_nic_setup_tc,
1320         .ndo_set_rx_mode        = hns3_nic_set_rx_mode,
1321         .ndo_udp_tunnel_add     = hns3_nic_udp_tunnel_add,
1322         .ndo_udp_tunnel_del     = hns3_nic_udp_tunnel_del,
1323         .ndo_vlan_rx_add_vid    = hns3_vlan_rx_add_vid,
1324         .ndo_vlan_rx_kill_vid   = hns3_vlan_rx_kill_vid,
1325         .ndo_set_vf_vlan        = hns3_ndo_set_vf_vlan,
1326 };
1327
1328 /* hns3_probe - Device initialization routine
1329  * @pdev: PCI device information struct
1330  * @ent: entry in hns3_pci_tbl
1331  *
1332  * hns3_probe initializes a PF identified by a pci_dev structure.
1333  * The OS initialization, configuring of the PF private structure,
1334  * and a hardware reset occur.
1335  *
1336  * Returns 0 on success, negative on failure
1337  */
1338 static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1339 {
1340         struct hnae3_ae_dev *ae_dev;
1341         int ret;
1342
1343         ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1344                               GFP_KERNEL);
1345         if (!ae_dev) {
1346                 ret = -ENOMEM;
1347                 return ret;
1348         }
1349
1350         ae_dev->pdev = pdev;
1351         ae_dev->dev_type = HNAE3_DEV_KNIC;
1352         pci_set_drvdata(pdev, ae_dev);
1353
1354         return hnae3_register_ae_dev(ae_dev);
1355 }
1356
1357 /* hns3_remove - Device removal routine
1358  * @pdev: PCI device information struct
1359  */
1360 static void hns3_remove(struct pci_dev *pdev)
1361 {
1362         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1363
1364         hnae3_unregister_ae_dev(ae_dev);
1365
1366         devm_kfree(&pdev->dev, ae_dev);
1367
1368         pci_set_drvdata(pdev, NULL);
1369 }
1370
1371 static struct pci_driver hns3_driver = {
1372         .name     = hns3_driver_name,
1373         .id_table = hns3_pci_tbl,
1374         .probe    = hns3_probe,
1375         .remove   = hns3_remove,
1376 };
1377
1378 /* set default feature to hns3 */
1379 static void hns3_set_default_feature(struct net_device *netdev)
1380 {
1381         netdev->priv_flags |= IFF_UNICAST_FLT;
1382
1383         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1384                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1385                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1386                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1387                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1388
1389         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1390
1391         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1392
1393         netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1394                 NETIF_F_HW_VLAN_CTAG_FILTER |
1395                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1396                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1397                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1398                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1399
1400         netdev->vlan_features |=
1401                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1402                 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1403                 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1404                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1405                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1406
1407         netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1408                 NETIF_F_HW_VLAN_CTAG_FILTER |
1409                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1410                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1411                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1412                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1413 }
1414
1415 static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1416                              struct hns3_desc_cb *cb)
1417 {
1418         unsigned int order = hnae_page_order(ring);
1419         struct page *p;
1420
1421         p = dev_alloc_pages(order);
1422         if (!p)
1423                 return -ENOMEM;
1424
1425         cb->priv = p;
1426         cb->page_offset = 0;
1427         cb->reuse_flag = 0;
1428         cb->buf  = page_address(p);
1429         cb->length = hnae_page_size(ring);
1430         cb->type = DESC_TYPE_PAGE;
1431
1432         memset(cb->buf, 0, cb->length);
1433
1434         return 0;
1435 }
1436
1437 static void hns3_free_buffer(struct hns3_enet_ring *ring,
1438                              struct hns3_desc_cb *cb)
1439 {
1440         if (cb->type == DESC_TYPE_SKB)
1441                 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1442         else if (!HNAE3_IS_TX_RING(ring))
1443                 put_page((struct page *)cb->priv);
1444         memset(cb, 0, sizeof(*cb));
1445 }
1446
1447 static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1448 {
1449         cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1450                                cb->length, ring_to_dma_dir(ring));
1451
1452         if (dma_mapping_error(ring_to_dev(ring), cb->dma))
1453                 return -EIO;
1454
1455         return 0;
1456 }
1457
1458 static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1459                               struct hns3_desc_cb *cb)
1460 {
1461         if (cb->type == DESC_TYPE_SKB)
1462                 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1463                                  ring_to_dma_dir(ring));
1464         else
1465                 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1466                                ring_to_dma_dir(ring));
1467 }
1468
1469 static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1470 {
1471         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1472         ring->desc[i].addr = 0;
1473 }
1474
1475 static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1476 {
1477         struct hns3_desc_cb *cb = &ring->desc_cb[i];
1478
1479         if (!ring->desc_cb[i].dma)
1480                 return;
1481
1482         hns3_buffer_detach(ring, i);
1483         hns3_free_buffer(ring, cb);
1484 }
1485
1486 static void hns3_free_buffers(struct hns3_enet_ring *ring)
1487 {
1488         int i;
1489
1490         for (i = 0; i < ring->desc_num; i++)
1491                 hns3_free_buffer_detach(ring, i);
1492 }
1493
1494 /* free desc along with its attached buffer */
1495 static void hns3_free_desc(struct hns3_enet_ring *ring)
1496 {
1497         hns3_free_buffers(ring);
1498
1499         dma_unmap_single(ring_to_dev(ring), ring->desc_dma_addr,
1500                          ring->desc_num * sizeof(ring->desc[0]),
1501                          DMA_BIDIRECTIONAL);
1502         ring->desc_dma_addr = 0;
1503         kfree(ring->desc);
1504         ring->desc = NULL;
1505 }
1506
1507 static int hns3_alloc_desc(struct hns3_enet_ring *ring)
1508 {
1509         int size = ring->desc_num * sizeof(ring->desc[0]);
1510
1511         ring->desc = kzalloc(size, GFP_KERNEL);
1512         if (!ring->desc)
1513                 return -ENOMEM;
1514
1515         ring->desc_dma_addr = dma_map_single(ring_to_dev(ring), ring->desc,
1516                                              size, DMA_BIDIRECTIONAL);
1517         if (dma_mapping_error(ring_to_dev(ring), ring->desc_dma_addr)) {
1518                 ring->desc_dma_addr = 0;
1519                 kfree(ring->desc);
1520                 ring->desc = NULL;
1521                 return -ENOMEM;
1522         }
1523
1524         return 0;
1525 }
1526
1527 static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
1528                                    struct hns3_desc_cb *cb)
1529 {
1530         int ret;
1531
1532         ret = hns3_alloc_buffer(ring, cb);
1533         if (ret)
1534                 goto out;
1535
1536         ret = hns3_map_buffer(ring, cb);
1537         if (ret)
1538                 goto out_with_buf;
1539
1540         return 0;
1541
1542 out_with_buf:
1543         hns3_free_buffers(ring);
1544 out:
1545         return ret;
1546 }
1547
1548 static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
1549 {
1550         int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
1551
1552         if (ret)
1553                 return ret;
1554
1555         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1556
1557         return 0;
1558 }
1559
1560 /* Allocate memory for raw pkg, and map with dma */
1561 static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
1562 {
1563         int i, j, ret;
1564
1565         for (i = 0; i < ring->desc_num; i++) {
1566                 ret = hns3_alloc_buffer_attach(ring, i);
1567                 if (ret)
1568                         goto out_buffer_fail;
1569         }
1570
1571         return 0;
1572
1573 out_buffer_fail:
1574         for (j = i - 1; j >= 0; j--)
1575                 hns3_free_buffer_detach(ring, j);
1576         return ret;
1577 }
1578
1579 /* detach a in-used buffer and replace with a reserved one  */
1580 static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
1581                                 struct hns3_desc_cb *res_cb)
1582 {
1583         hns3_map_buffer(ring, &ring->desc_cb[i]);
1584         ring->desc_cb[i] = *res_cb;
1585         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1586 }
1587
1588 static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
1589 {
1590         ring->desc_cb[i].reuse_flag = 0;
1591         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
1592                 + ring->desc_cb[i].page_offset);
1593 }
1594
1595 static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
1596                                       int *pkts)
1597 {
1598         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
1599
1600         (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
1601         (*bytes) += desc_cb->length;
1602         /* desc_cb will be cleaned, after hnae_free_buffer_detach*/
1603         hns3_free_buffer_detach(ring, ring->next_to_clean);
1604
1605         ring_ptr_move_fw(ring, next_to_clean);
1606 }
1607
1608 static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
1609 {
1610         int u = ring->next_to_use;
1611         int c = ring->next_to_clean;
1612
1613         if (unlikely(h > ring->desc_num))
1614                 return 0;
1615
1616         return u > c ? (h > c && h <= u) : (h > c || h <= u);
1617 }
1618
1619 int hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
1620 {
1621         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1622         struct netdev_queue *dev_queue;
1623         int bytes, pkts;
1624         int head;
1625
1626         head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
1627         rmb(); /* Make sure head is ready before touch any data */
1628
1629         if (is_ring_empty(ring) || head == ring->next_to_clean)
1630                 return 0; /* no data to poll */
1631
1632         if (!is_valid_clean_head(ring, head)) {
1633                 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
1634                            ring->next_to_use, ring->next_to_clean);
1635
1636                 u64_stats_update_begin(&ring->syncp);
1637                 ring->stats.io_err_cnt++;
1638                 u64_stats_update_end(&ring->syncp);
1639                 return -EIO;
1640         }
1641
1642         bytes = 0;
1643         pkts = 0;
1644         while (head != ring->next_to_clean && budget) {
1645                 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
1646                 /* Issue prefetch for next Tx descriptor */
1647                 prefetch(&ring->desc_cb[ring->next_to_clean]);
1648                 budget--;
1649         }
1650
1651         ring->tqp_vector->tx_group.total_bytes += bytes;
1652         ring->tqp_vector->tx_group.total_packets += pkts;
1653
1654         u64_stats_update_begin(&ring->syncp);
1655         ring->stats.tx_bytes += bytes;
1656         ring->stats.tx_pkts += pkts;
1657         u64_stats_update_end(&ring->syncp);
1658
1659         dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
1660         netdev_tx_completed_queue(dev_queue, pkts, bytes);
1661
1662         if (unlikely(pkts && netif_carrier_ok(netdev) &&
1663                      (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
1664                 /* Make sure that anybody stopping the queue after this
1665                  * sees the new next_to_clean.
1666                  */
1667                 smp_mb();
1668                 if (netif_tx_queue_stopped(dev_queue)) {
1669                         netif_tx_wake_queue(dev_queue);
1670                         ring->stats.restart_queue++;
1671                 }
1672         }
1673
1674         return !!budget;
1675 }
1676
1677 static int hns3_desc_unused(struct hns3_enet_ring *ring)
1678 {
1679         int ntc = ring->next_to_clean;
1680         int ntu = ring->next_to_use;
1681
1682         return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
1683 }
1684
1685 static void
1686 hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
1687 {
1688         struct hns3_desc_cb *desc_cb;
1689         struct hns3_desc_cb res_cbs;
1690         int i, ret;
1691
1692         for (i = 0; i < cleand_count; i++) {
1693                 desc_cb = &ring->desc_cb[ring->next_to_use];
1694                 if (desc_cb->reuse_flag) {
1695                         u64_stats_update_begin(&ring->syncp);
1696                         ring->stats.reuse_pg_cnt++;
1697                         u64_stats_update_end(&ring->syncp);
1698
1699                         hns3_reuse_buffer(ring, ring->next_to_use);
1700                 } else {
1701                         ret = hns3_reserve_buffer_map(ring, &res_cbs);
1702                         if (ret) {
1703                                 u64_stats_update_begin(&ring->syncp);
1704                                 ring->stats.sw_err_cnt++;
1705                                 u64_stats_update_end(&ring->syncp);
1706
1707                                 netdev_err(ring->tqp->handle->kinfo.netdev,
1708                                            "hnae reserve buffer map failed.\n");
1709                                 break;
1710                         }
1711                         hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
1712                 }
1713
1714                 ring_ptr_move_fw(ring, next_to_use);
1715         }
1716
1717         wmb(); /* Make all data has been write before submit */
1718         writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
1719 }
1720
1721 /* hns3_nic_get_headlen - determine size of header for LRO/GRO
1722  * @data: pointer to the start of the headers
1723  * @max: total length of section to find headers in
1724  *
1725  * This function is meant to determine the length of headers that will
1726  * be recognized by hardware for LRO, GRO, and RSC offloads.  The main
1727  * motivation of doing this is to only perform one pull for IPv4 TCP
1728  * packets so that we can do basic things like calculating the gso_size
1729  * based on the average data per packet.
1730  */
1731 static unsigned int hns3_nic_get_headlen(unsigned char *data, u32 flag,
1732                                          unsigned int max_size)
1733 {
1734         unsigned char *network;
1735         u8 hlen;
1736
1737         /* This should never happen, but better safe than sorry */
1738         if (max_size < ETH_HLEN)
1739                 return max_size;
1740
1741         /* Initialize network frame pointer */
1742         network = data;
1743
1744         /* Set first protocol and move network header forward */
1745         network += ETH_HLEN;
1746
1747         /* Handle any vlan tag if present */
1748         if (hnae_get_field(flag, HNS3_RXD_VLAN_M, HNS3_RXD_VLAN_S)
1749                 == HNS3_RX_FLAG_VLAN_PRESENT) {
1750                 if ((typeof(max_size))(network - data) > (max_size - VLAN_HLEN))
1751                         return max_size;
1752
1753                 network += VLAN_HLEN;
1754         }
1755
1756         /* Handle L3 protocols */
1757         if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
1758                 == HNS3_RX_FLAG_L3ID_IPV4) {
1759                 if ((typeof(max_size))(network - data) >
1760                     (max_size - sizeof(struct iphdr)))
1761                         return max_size;
1762
1763                 /* Access ihl as a u8 to avoid unaligned access on ia64 */
1764                 hlen = (network[0] & 0x0F) << 2;
1765
1766                 /* Verify hlen meets minimum size requirements */
1767                 if (hlen < sizeof(struct iphdr))
1768                         return network - data;
1769
1770                 /* Record next protocol if header is present */
1771         } else if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
1772                 == HNS3_RX_FLAG_L3ID_IPV6) {
1773                 if ((typeof(max_size))(network - data) >
1774                     (max_size - sizeof(struct ipv6hdr)))
1775                         return max_size;
1776
1777                 /* Record next protocol */
1778                 hlen = sizeof(struct ipv6hdr);
1779         } else {
1780                 return network - data;
1781         }
1782
1783         /* Relocate pointer to start of L4 header */
1784         network += hlen;
1785
1786         /* Finally sort out TCP/UDP */
1787         if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
1788                 == HNS3_RX_FLAG_L4ID_TCP) {
1789                 if ((typeof(max_size))(network - data) >
1790                     (max_size - sizeof(struct tcphdr)))
1791                         return max_size;
1792
1793                 /* Access doff as a u8 to avoid unaligned access on ia64 */
1794                 hlen = (network[12] & 0xF0) >> 2;
1795
1796                 /* Verify hlen meets minimum size requirements */
1797                 if (hlen < sizeof(struct tcphdr))
1798                         return network - data;
1799
1800                 network += hlen;
1801         } else if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
1802                 == HNS3_RX_FLAG_L4ID_UDP) {
1803                 if ((typeof(max_size))(network - data) >
1804                     (max_size - sizeof(struct udphdr)))
1805                         return max_size;
1806
1807                 network += sizeof(struct udphdr);
1808         }
1809
1810         /* If everything has gone correctly network should be the
1811          * data section of the packet and will be the end of the header.
1812          * If not then it probably represents the end of the last recognized
1813          * header.
1814          */
1815         if ((typeof(max_size))(network - data) < max_size)
1816                 return network - data;
1817         else
1818                 return max_size;
1819 }
1820
1821 static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
1822                                 struct hns3_enet_ring *ring, int pull_len,
1823                                 struct hns3_desc_cb *desc_cb)
1824 {
1825         struct hns3_desc *desc;
1826         int truesize, size;
1827         int last_offset;
1828         bool twobufs;
1829
1830         twobufs = ((PAGE_SIZE < 8192) &&
1831                 hnae_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
1832
1833         desc = &ring->desc[ring->next_to_clean];
1834         size = le16_to_cpu(desc->rx.size);
1835
1836         if (twobufs) {
1837                 truesize = hnae_buf_size(ring);
1838         } else {
1839                 truesize = ALIGN(size, L1_CACHE_BYTES);
1840                 last_offset = hnae_page_size(ring) - hnae_buf_size(ring);
1841         }
1842
1843         skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
1844                         size - pull_len, truesize - pull_len);
1845
1846          /* Avoid re-using remote pages,flag default unreuse */
1847         if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
1848                 return;
1849
1850         if (twobufs) {
1851                 /* If we are only owner of page we can reuse it */
1852                 if (likely(page_count(desc_cb->priv) == 1)) {
1853                         /* Flip page offset to other buffer */
1854                         desc_cb->page_offset ^= truesize;
1855
1856                         desc_cb->reuse_flag = 1;
1857                         /* bump ref count on page before it is given*/
1858                         get_page(desc_cb->priv);
1859                 }
1860                 return;
1861         }
1862
1863         /* Move offset up to the next cache line */
1864         desc_cb->page_offset += truesize;
1865
1866         if (desc_cb->page_offset <= last_offset) {
1867                 desc_cb->reuse_flag = 1;
1868                 /* Bump ref count on page before it is given*/
1869                 get_page(desc_cb->priv);
1870         }
1871 }
1872
1873 static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
1874                              struct hns3_desc *desc)
1875 {
1876         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1877         int l3_type, l4_type;
1878         u32 bd_base_info;
1879         int ol4_type;
1880         u32 l234info;
1881
1882         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
1883         l234info = le32_to_cpu(desc->rx.l234_info);
1884
1885         skb->ip_summed = CHECKSUM_NONE;
1886
1887         skb_checksum_none_assert(skb);
1888
1889         if (!(netdev->features & NETIF_F_RXCSUM))
1890                 return;
1891
1892         /* check if hardware has done checksum */
1893         if (!hnae_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
1894                 return;
1895
1896         if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L3E_B) ||
1897                      hnae_get_bit(l234info, HNS3_RXD_L4E_B) ||
1898                      hnae_get_bit(l234info, HNS3_RXD_OL3E_B) ||
1899                      hnae_get_bit(l234info, HNS3_RXD_OL4E_B))) {
1900                 netdev_err(netdev, "L3/L4 error pkt\n");
1901                 u64_stats_update_begin(&ring->syncp);
1902                 ring->stats.l3l4_csum_err++;
1903                 u64_stats_update_end(&ring->syncp);
1904
1905                 return;
1906         }
1907
1908         l3_type = hnae_get_field(l234info, HNS3_RXD_L3ID_M,
1909                                  HNS3_RXD_L3ID_S);
1910         l4_type = hnae_get_field(l234info, HNS3_RXD_L4ID_M,
1911                                  HNS3_RXD_L4ID_S);
1912
1913         ol4_type = hnae_get_field(l234info, HNS3_RXD_OL4ID_M, HNS3_RXD_OL4ID_S);
1914         switch (ol4_type) {
1915         case HNS3_OL4_TYPE_MAC_IN_UDP:
1916         case HNS3_OL4_TYPE_NVGRE:
1917                 skb->csum_level = 1;
1918         case HNS3_OL4_TYPE_NO_TUN:
1919                 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
1920                 if (l3_type == HNS3_L3_TYPE_IPV4 ||
1921                     (l3_type == HNS3_L3_TYPE_IPV6 &&
1922                      (l4_type == HNS3_L4_TYPE_UDP ||
1923                       l4_type == HNS3_L4_TYPE_TCP ||
1924                       l4_type == HNS3_L4_TYPE_SCTP)))
1925                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1926                 break;
1927         }
1928 }
1929
1930 static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
1931                              struct sk_buff **out_skb, int *out_bnum)
1932 {
1933         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1934         struct hns3_desc_cb *desc_cb;
1935         struct hns3_desc *desc;
1936         struct sk_buff *skb;
1937         unsigned char *va;
1938         u32 bd_base_info;
1939         int pull_len;
1940         u32 l234info;
1941         int length;
1942         int bnum;
1943
1944         desc = &ring->desc[ring->next_to_clean];
1945         desc_cb = &ring->desc_cb[ring->next_to_clean];
1946
1947         prefetch(desc);
1948
1949         length = le16_to_cpu(desc->rx.pkt_len);
1950         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
1951         l234info = le32_to_cpu(desc->rx.l234_info);
1952
1953         /* Check valid BD */
1954         if (!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))
1955                 return -EFAULT;
1956
1957         va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
1958
1959         /* Prefetch first cache line of first page
1960          * Idea is to cache few bytes of the header of the packet. Our L1 Cache
1961          * line size is 64B so need to prefetch twice to make it 128B. But in
1962          * actual we can have greater size of caches with 128B Level 1 cache
1963          * lines. In such a case, single fetch would suffice to cache in the
1964          * relevant part of the header.
1965          */
1966         prefetch(va);
1967 #if L1_CACHE_BYTES < 128
1968         prefetch(va + L1_CACHE_BYTES);
1969 #endif
1970
1971         skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
1972                                         HNS3_RX_HEAD_SIZE);
1973         if (unlikely(!skb)) {
1974                 netdev_err(netdev, "alloc rx skb fail\n");
1975
1976                 u64_stats_update_begin(&ring->syncp);
1977                 ring->stats.sw_err_cnt++;
1978                 u64_stats_update_end(&ring->syncp);
1979
1980                 return -ENOMEM;
1981         }
1982
1983         prefetchw(skb->data);
1984
1985         bnum = 1;
1986         if (length <= HNS3_RX_HEAD_SIZE) {
1987                 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
1988
1989                 /* We can reuse buffer as-is, just make sure it is local */
1990                 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
1991                         desc_cb->reuse_flag = 1;
1992                 else /* This page cannot be reused so discard it */
1993                         put_page(desc_cb->priv);
1994
1995                 ring_ptr_move_fw(ring, next_to_clean);
1996         } else {
1997                 u64_stats_update_begin(&ring->syncp);
1998                 ring->stats.seg_pkt_cnt++;
1999                 u64_stats_update_end(&ring->syncp);
2000
2001                 pull_len = hns3_nic_get_headlen(va, l234info,
2002                                                 HNS3_RX_HEAD_SIZE);
2003                 memcpy(__skb_put(skb, pull_len), va,
2004                        ALIGN(pull_len, sizeof(long)));
2005
2006                 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
2007                 ring_ptr_move_fw(ring, next_to_clean);
2008
2009                 while (!hnae_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2010                         desc = &ring->desc[ring->next_to_clean];
2011                         desc_cb = &ring->desc_cb[ring->next_to_clean];
2012                         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2013                         hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
2014                         ring_ptr_move_fw(ring, next_to_clean);
2015                         bnum++;
2016                 }
2017         }
2018
2019         *out_bnum = bnum;
2020
2021         if (unlikely(!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
2022                 netdev_err(netdev, "no valid bd,%016llx,%016llx\n",
2023                            ((u64 *)desc)[0], ((u64 *)desc)[1]);
2024                 u64_stats_update_begin(&ring->syncp);
2025                 ring->stats.non_vld_descs++;
2026                 u64_stats_update_end(&ring->syncp);
2027
2028                 dev_kfree_skb_any(skb);
2029                 return -EINVAL;
2030         }
2031
2032         if (unlikely((!desc->rx.pkt_len) ||
2033                      hnae_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
2034                 netdev_err(netdev, "truncated pkt\n");
2035                 u64_stats_update_begin(&ring->syncp);
2036                 ring->stats.err_pkt_len++;
2037                 u64_stats_update_end(&ring->syncp);
2038
2039                 dev_kfree_skb_any(skb);
2040                 return -EFAULT;
2041         }
2042
2043         if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L2E_B))) {
2044                 netdev_err(netdev, "L2 error pkt\n");
2045                 u64_stats_update_begin(&ring->syncp);
2046                 ring->stats.l2_err++;
2047                 u64_stats_update_end(&ring->syncp);
2048
2049                 dev_kfree_skb_any(skb);
2050                 return -EFAULT;
2051         }
2052
2053         u64_stats_update_begin(&ring->syncp);
2054         ring->stats.rx_pkts++;
2055         ring->stats.rx_bytes += skb->len;
2056         u64_stats_update_end(&ring->syncp);
2057
2058         ring->tqp_vector->rx_group.total_bytes += skb->len;
2059
2060         hns3_rx_checksum(ring, skb, desc);
2061         return 0;
2062 }
2063
2064 static int hns3_clean_rx_ring(struct hns3_enet_ring *ring, int budget)
2065 {
2066 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2067         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2068         int recv_pkts, recv_bds, clean_count, err;
2069         int unused_count = hns3_desc_unused(ring);
2070         struct sk_buff *skb = NULL;
2071         int num, bnum = 0;
2072
2073         num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2074         rmb(); /* Make sure num taken effect before the other data is touched */
2075
2076         recv_pkts = 0, recv_bds = 0, clean_count = 0;
2077         num -= unused_count;
2078
2079         while (recv_pkts < budget && recv_bds < num) {
2080                 /* Reuse or realloc buffers */
2081                 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2082                         hns3_nic_alloc_rx_buffers(ring,
2083                                                   clean_count + unused_count);
2084                         clean_count = 0;
2085                         unused_count = hns3_desc_unused(ring);
2086                 }
2087
2088                 /* Poll one pkt */
2089                 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2090                 if (unlikely(!skb)) /* This fault cannot be repaired */
2091                         goto out;
2092
2093                 recv_bds += bnum;
2094                 clean_count += bnum;
2095                 if (unlikely(err)) {  /* Do jump the err */
2096                         recv_pkts++;
2097                         continue;
2098                 }
2099
2100                 /* Do update ip stack process */
2101                 skb->protocol = eth_type_trans(skb, netdev);
2102                 (void)napi_gro_receive(&ring->tqp_vector->napi, skb);
2103
2104                 recv_pkts++;
2105         }
2106
2107 out:
2108         /* Make all data has been write before submit */
2109         if (clean_count + unused_count > 0)
2110                 hns3_nic_alloc_rx_buffers(ring,
2111                                           clean_count + unused_count);
2112
2113         return recv_pkts;
2114 }
2115
2116 static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2117 {
2118 #define HNS3_RX_ULTRA_PACKET_RATE 40000
2119         enum hns3_flow_level_range new_flow_level;
2120         struct hns3_enet_tqp_vector *tqp_vector;
2121         int packets_per_secs;
2122         int bytes_per_usecs;
2123         u16 new_int_gl;
2124         int usecs;
2125
2126         if (!ring_group->int_gl)
2127                 return false;
2128
2129         if (ring_group->total_packets == 0) {
2130                 ring_group->int_gl = HNS3_INT_GL_50K;
2131                 ring_group->flow_level = HNS3_FLOW_LOW;
2132                 return true;
2133         }
2134
2135         /* Simple throttlerate management
2136          * 0-10MB/s   lower     (50000 ints/s)
2137          * 10-20MB/s   middle    (20000 ints/s)
2138          * 20-1249MB/s high      (18000 ints/s)
2139          * > 40000pps  ultra     (8000 ints/s)
2140          */
2141         new_flow_level = ring_group->flow_level;
2142         new_int_gl = ring_group->int_gl;
2143         tqp_vector = ring_group->ring->tqp_vector;
2144         usecs = (ring_group->int_gl << 1);
2145         bytes_per_usecs = ring_group->total_bytes / usecs;
2146         /* 1000000 microseconds */
2147         packets_per_secs = ring_group->total_packets * 1000000 / usecs;
2148
2149         switch (new_flow_level) {
2150         case HNS3_FLOW_LOW:
2151                 if (bytes_per_usecs > 10)
2152                         new_flow_level = HNS3_FLOW_MID;
2153                 break;
2154         case HNS3_FLOW_MID:
2155                 if (bytes_per_usecs > 20)
2156                         new_flow_level = HNS3_FLOW_HIGH;
2157                 else if (bytes_per_usecs <= 10)
2158                         new_flow_level = HNS3_FLOW_LOW;
2159                 break;
2160         case HNS3_FLOW_HIGH:
2161         case HNS3_FLOW_ULTRA:
2162         default:
2163                 if (bytes_per_usecs <= 20)
2164                         new_flow_level = HNS3_FLOW_MID;
2165                 break;
2166         }
2167
2168         if ((packets_per_secs > HNS3_RX_ULTRA_PACKET_RATE) &&
2169             (&tqp_vector->rx_group == ring_group))
2170                 new_flow_level = HNS3_FLOW_ULTRA;
2171
2172         switch (new_flow_level) {
2173         case HNS3_FLOW_LOW:
2174                 new_int_gl = HNS3_INT_GL_50K;
2175                 break;
2176         case HNS3_FLOW_MID:
2177                 new_int_gl = HNS3_INT_GL_20K;
2178                 break;
2179         case HNS3_FLOW_HIGH:
2180                 new_int_gl = HNS3_INT_GL_18K;
2181                 break;
2182         case HNS3_FLOW_ULTRA:
2183                 new_int_gl = HNS3_INT_GL_8K;
2184                 break;
2185         default:
2186                 break;
2187         }
2188
2189         ring_group->total_bytes = 0;
2190         ring_group->total_packets = 0;
2191         ring_group->flow_level = new_flow_level;
2192         if (new_int_gl != ring_group->int_gl) {
2193                 ring_group->int_gl = new_int_gl;
2194                 return true;
2195         }
2196         return false;
2197 }
2198
2199 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2200 {
2201         u16 rx_int_gl, tx_int_gl;
2202         bool rx, tx;
2203
2204         rx = hns3_get_new_int_gl(&tqp_vector->rx_group);
2205         tx = hns3_get_new_int_gl(&tqp_vector->tx_group);
2206         rx_int_gl = tqp_vector->rx_group.int_gl;
2207         tx_int_gl = tqp_vector->tx_group.int_gl;
2208         if (rx && tx) {
2209                 if (rx_int_gl > tx_int_gl) {
2210                         tqp_vector->tx_group.int_gl = rx_int_gl;
2211                         tqp_vector->tx_group.flow_level =
2212                                 tqp_vector->rx_group.flow_level;
2213                         hns3_set_vector_coalesc_gl(tqp_vector, rx_int_gl);
2214                 } else {
2215                         tqp_vector->rx_group.int_gl = tx_int_gl;
2216                         tqp_vector->rx_group.flow_level =
2217                                 tqp_vector->tx_group.flow_level;
2218                         hns3_set_vector_coalesc_gl(tqp_vector, tx_int_gl);
2219                 }
2220         }
2221 }
2222
2223 static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2224 {
2225         struct hns3_enet_ring *ring;
2226         int rx_pkt_total = 0;
2227
2228         struct hns3_enet_tqp_vector *tqp_vector =
2229                 container_of(napi, struct hns3_enet_tqp_vector, napi);
2230         bool clean_complete = true;
2231         int rx_budget;
2232
2233         /* Since the actual Tx work is minimal, we can give the Tx a larger
2234          * budget and be more aggressive about cleaning up the Tx descriptors.
2235          */
2236         hns3_for_each_ring(ring, tqp_vector->tx_group) {
2237                 if (!hns3_clean_tx_ring(ring, budget))
2238                         clean_complete = false;
2239         }
2240
2241         /* make sure rx ring budget not smaller than 1 */
2242         rx_budget = max(budget / tqp_vector->num_tqps, 1);
2243
2244         hns3_for_each_ring(ring, tqp_vector->rx_group) {
2245                 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget);
2246
2247                 if (rx_cleaned >= rx_budget)
2248                         clean_complete = false;
2249
2250                 rx_pkt_total += rx_cleaned;
2251         }
2252
2253         tqp_vector->rx_group.total_packets += rx_pkt_total;
2254
2255         if (!clean_complete)
2256                 return budget;
2257
2258         napi_complete(napi);
2259         hns3_update_new_int_gl(tqp_vector);
2260         hns3_mask_vector_irq(tqp_vector, 1);
2261
2262         return rx_pkt_total;
2263 }
2264
2265 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2266                                       struct hnae3_ring_chain_node *head)
2267 {
2268         struct pci_dev *pdev = tqp_vector->handle->pdev;
2269         struct hnae3_ring_chain_node *cur_chain = head;
2270         struct hnae3_ring_chain_node *chain;
2271         struct hns3_enet_ring *tx_ring;
2272         struct hns3_enet_ring *rx_ring;
2273
2274         tx_ring = tqp_vector->tx_group.ring;
2275         if (tx_ring) {
2276                 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2277                 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2278                              HNAE3_RING_TYPE_TX);
2279
2280                 cur_chain->next = NULL;
2281
2282                 while (tx_ring->next) {
2283                         tx_ring = tx_ring->next;
2284
2285                         chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2286                                              GFP_KERNEL);
2287                         if (!chain)
2288                                 return -ENOMEM;
2289
2290                         cur_chain->next = chain;
2291                         chain->tqp_index = tx_ring->tqp->tqp_index;
2292                         hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2293                                      HNAE3_RING_TYPE_TX);
2294
2295                         cur_chain = chain;
2296                 }
2297         }
2298
2299         rx_ring = tqp_vector->rx_group.ring;
2300         if (!tx_ring && rx_ring) {
2301                 cur_chain->next = NULL;
2302                 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2303                 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2304                              HNAE3_RING_TYPE_RX);
2305
2306                 rx_ring = rx_ring->next;
2307         }
2308
2309         while (rx_ring) {
2310                 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2311                 if (!chain)
2312                         return -ENOMEM;
2313
2314                 cur_chain->next = chain;
2315                 chain->tqp_index = rx_ring->tqp->tqp_index;
2316                 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2317                              HNAE3_RING_TYPE_RX);
2318                 cur_chain = chain;
2319
2320                 rx_ring = rx_ring->next;
2321         }
2322
2323         return 0;
2324 }
2325
2326 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2327                                         struct hnae3_ring_chain_node *head)
2328 {
2329         struct pci_dev *pdev = tqp_vector->handle->pdev;
2330         struct hnae3_ring_chain_node *chain_tmp, *chain;
2331
2332         chain = head->next;
2333
2334         while (chain) {
2335                 chain_tmp = chain->next;
2336                 devm_kfree(&pdev->dev, chain);
2337                 chain = chain_tmp;
2338         }
2339 }
2340
2341 static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2342                                    struct hns3_enet_ring *ring)
2343 {
2344         ring->next = group->ring;
2345         group->ring = ring;
2346
2347         group->count++;
2348 }
2349
2350 static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2351 {
2352         struct hnae3_ring_chain_node vector_ring_chain;
2353         struct hnae3_handle *h = priv->ae_handle;
2354         struct hns3_enet_tqp_vector *tqp_vector;
2355         struct hnae3_vector_info *vector;
2356         struct pci_dev *pdev = h->pdev;
2357         u16 tqp_num = h->kinfo.num_tqps;
2358         u16 vector_num;
2359         int ret = 0;
2360         u16 i;
2361
2362         /* RSS size, cpu online and vector_num should be the same */
2363         /* Should consider 2p/4p later */
2364         vector_num = min_t(u16, num_online_cpus(), tqp_num);
2365         vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2366                               GFP_KERNEL);
2367         if (!vector)
2368                 return -ENOMEM;
2369
2370         vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2371
2372         priv->vector_num = vector_num;
2373         priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2374                 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2375                              GFP_KERNEL);
2376         if (!priv->tqp_vector)
2377                 return -ENOMEM;
2378
2379         for (i = 0; i < tqp_num; i++) {
2380                 u16 vector_i = i % vector_num;
2381
2382                 tqp_vector = &priv->tqp_vector[vector_i];
2383
2384                 hns3_add_ring_to_group(&tqp_vector->tx_group,
2385                                        priv->ring_data[i].ring);
2386
2387                 hns3_add_ring_to_group(&tqp_vector->rx_group,
2388                                        priv->ring_data[i + tqp_num].ring);
2389
2390                 tqp_vector->idx = vector_i;
2391                 tqp_vector->mask_addr = vector[vector_i].io_addr;
2392                 tqp_vector->vector_irq = vector[vector_i].vector;
2393                 tqp_vector->num_tqps++;
2394
2395                 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2396                 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
2397         }
2398
2399         for (i = 0; i < vector_num; i++) {
2400                 tqp_vector = &priv->tqp_vector[i];
2401
2402                 tqp_vector->rx_group.total_bytes = 0;
2403                 tqp_vector->rx_group.total_packets = 0;
2404                 tqp_vector->tx_group.total_bytes = 0;
2405                 tqp_vector->tx_group.total_packets = 0;
2406                 hns3_vector_gl_rl_init(tqp_vector);
2407                 tqp_vector->handle = h;
2408
2409                 ret = hns3_get_vector_ring_chain(tqp_vector,
2410                                                  &vector_ring_chain);
2411                 if (ret)
2412                         goto out;
2413
2414                 ret = h->ae_algo->ops->map_ring_to_vector(h,
2415                         tqp_vector->vector_irq, &vector_ring_chain);
2416                 if (ret)
2417                         goto out;
2418
2419                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2420
2421                 netif_napi_add(priv->netdev, &tqp_vector->napi,
2422                                hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2423         }
2424
2425 out:
2426         devm_kfree(&pdev->dev, vector);
2427         return ret;
2428 }
2429
2430 static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
2431 {
2432         struct hnae3_ring_chain_node vector_ring_chain;
2433         struct hnae3_handle *h = priv->ae_handle;
2434         struct hns3_enet_tqp_vector *tqp_vector;
2435         struct pci_dev *pdev = h->pdev;
2436         int i, ret;
2437
2438         for (i = 0; i < priv->vector_num; i++) {
2439                 tqp_vector = &priv->tqp_vector[i];
2440
2441                 ret = hns3_get_vector_ring_chain(tqp_vector,
2442                                                  &vector_ring_chain);
2443                 if (ret)
2444                         return ret;
2445
2446                 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
2447                         tqp_vector->vector_irq, &vector_ring_chain);
2448                 if (ret)
2449                         return ret;
2450
2451                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2452
2453                 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
2454                         (void)irq_set_affinity_hint(
2455                                 priv->tqp_vector[i].vector_irq,
2456                                                     NULL);
2457                         devm_free_irq(&pdev->dev,
2458                                       priv->tqp_vector[i].vector_irq,
2459                                       &priv->tqp_vector[i]);
2460                 }
2461
2462                 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
2463
2464                 netif_napi_del(&priv->tqp_vector[i].napi);
2465         }
2466
2467         devm_kfree(&pdev->dev, priv->tqp_vector);
2468
2469         return 0;
2470 }
2471
2472 static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
2473                              int ring_type)
2474 {
2475         struct hns3_nic_ring_data *ring_data = priv->ring_data;
2476         int queue_num = priv->ae_handle->kinfo.num_tqps;
2477         struct pci_dev *pdev = priv->ae_handle->pdev;
2478         struct hns3_enet_ring *ring;
2479
2480         ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
2481         if (!ring)
2482                 return -ENOMEM;
2483
2484         if (ring_type == HNAE3_RING_TYPE_TX) {
2485                 ring_data[q->tqp_index].ring = ring;
2486                 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
2487         } else {
2488                 ring_data[q->tqp_index + queue_num].ring = ring;
2489                 ring->io_base = q->io_base;
2490         }
2491
2492         hnae_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
2493
2494         ring_data[q->tqp_index].queue_index = q->tqp_index;
2495
2496         ring->tqp = q;
2497         ring->desc = NULL;
2498         ring->desc_cb = NULL;
2499         ring->dev = priv->dev;
2500         ring->desc_dma_addr = 0;
2501         ring->buf_size = q->buf_size;
2502         ring->desc_num = q->desc_num;
2503         ring->next_to_use = 0;
2504         ring->next_to_clean = 0;
2505
2506         return 0;
2507 }
2508
2509 static int hns3_queue_to_ring(struct hnae3_queue *tqp,
2510                               struct hns3_nic_priv *priv)
2511 {
2512         int ret;
2513
2514         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
2515         if (ret)
2516                 return ret;
2517
2518         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
2519         if (ret)
2520                 return ret;
2521
2522         return 0;
2523 }
2524
2525 static int hns3_get_ring_config(struct hns3_nic_priv *priv)
2526 {
2527         struct hnae3_handle *h = priv->ae_handle;
2528         struct pci_dev *pdev = h->pdev;
2529         int i, ret;
2530
2531         priv->ring_data =  devm_kzalloc(&pdev->dev, h->kinfo.num_tqps *
2532                                         sizeof(*priv->ring_data) * 2,
2533                                         GFP_KERNEL);
2534         if (!priv->ring_data)
2535                 return -ENOMEM;
2536
2537         for (i = 0; i < h->kinfo.num_tqps; i++) {
2538                 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
2539                 if (ret)
2540                         goto err;
2541         }
2542
2543         return 0;
2544 err:
2545         devm_kfree(&pdev->dev, priv->ring_data);
2546         return ret;
2547 }
2548
2549 static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
2550 {
2551         int ret;
2552
2553         if (ring->desc_num <= 0 || ring->buf_size <= 0)
2554                 return -EINVAL;
2555
2556         ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
2557                                 GFP_KERNEL);
2558         if (!ring->desc_cb) {
2559                 ret = -ENOMEM;
2560                 goto out;
2561         }
2562
2563         ret = hns3_alloc_desc(ring);
2564         if (ret)
2565                 goto out_with_desc_cb;
2566
2567         if (!HNAE3_IS_TX_RING(ring)) {
2568                 ret = hns3_alloc_ring_buffers(ring);
2569                 if (ret)
2570                         goto out_with_desc;
2571         }
2572
2573         return 0;
2574
2575 out_with_desc:
2576         hns3_free_desc(ring);
2577 out_with_desc_cb:
2578         kfree(ring->desc_cb);
2579         ring->desc_cb = NULL;
2580 out:
2581         return ret;
2582 }
2583
2584 static void hns3_fini_ring(struct hns3_enet_ring *ring)
2585 {
2586         hns3_free_desc(ring);
2587         kfree(ring->desc_cb);
2588         ring->desc_cb = NULL;
2589         ring->next_to_clean = 0;
2590         ring->next_to_use = 0;
2591 }
2592
2593 int hns3_buf_size2type(u32 buf_size)
2594 {
2595         int bd_size_type;
2596
2597         switch (buf_size) {
2598         case 512:
2599                 bd_size_type = HNS3_BD_SIZE_512_TYPE;
2600                 break;
2601         case 1024:
2602                 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
2603                 break;
2604         case 2048:
2605                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2606                 break;
2607         case 4096:
2608                 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
2609                 break;
2610         default:
2611                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2612         }
2613
2614         return bd_size_type;
2615 }
2616
2617 static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
2618 {
2619         dma_addr_t dma = ring->desc_dma_addr;
2620         struct hnae3_queue *q = ring->tqp;
2621
2622         if (!HNAE3_IS_TX_RING(ring)) {
2623                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
2624                                (u32)dma);
2625                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
2626                                (u32)((dma >> 31) >> 1));
2627
2628                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
2629                                hns3_buf_size2type(ring->buf_size));
2630                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
2631                                ring->desc_num / 8 - 1);
2632
2633         } else {
2634                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
2635                                (u32)dma);
2636                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
2637                                (u32)((dma >> 31) >> 1));
2638
2639                 hns3_write_dev(q, HNS3_RING_TX_RING_BD_LEN_REG,
2640                                hns3_buf_size2type(ring->buf_size));
2641                 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
2642                                ring->desc_num / 8 - 1);
2643         }
2644 }
2645
2646 static int hns3_init_all_ring(struct hns3_nic_priv *priv)
2647 {
2648         struct hnae3_handle *h = priv->ae_handle;
2649         int ring_num = h->kinfo.num_tqps * 2;
2650         int i, j;
2651         int ret;
2652
2653         for (i = 0; i < ring_num; i++) {
2654                 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
2655                 if (ret) {
2656                         dev_err(priv->dev,
2657                                 "Alloc ring memory fail! ret=%d\n", ret);
2658                         goto out_when_alloc_ring_memory;
2659                 }
2660
2661                 hns3_init_ring_hw(priv->ring_data[i].ring);
2662
2663                 u64_stats_init(&priv->ring_data[i].ring->syncp);
2664         }
2665
2666         return 0;
2667
2668 out_when_alloc_ring_memory:
2669         for (j = i - 1; j >= 0; j--)
2670                 hns3_fini_ring(priv->ring_data[i].ring);
2671
2672         return -ENOMEM;
2673 }
2674
2675 static int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
2676 {
2677         struct hnae3_handle *h = priv->ae_handle;
2678         int i;
2679
2680         for (i = 0; i < h->kinfo.num_tqps; i++) {
2681                 if (h->ae_algo->ops->reset_queue)
2682                         h->ae_algo->ops->reset_queue(h, i);
2683
2684                 hns3_fini_ring(priv->ring_data[i].ring);
2685                 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
2686         }
2687
2688         return 0;
2689 }
2690
2691 /* Set mac addr if it is configured. or leave it to the AE driver */
2692 static void hns3_init_mac_addr(struct net_device *netdev)
2693 {
2694         struct hns3_nic_priv *priv = netdev_priv(netdev);
2695         struct hnae3_handle *h = priv->ae_handle;
2696         u8 mac_addr_temp[ETH_ALEN];
2697
2698         if (h->ae_algo->ops->get_mac_addr) {
2699                 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
2700                 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
2701         }
2702
2703         /* Check if the MAC address is valid, if not get a random one */
2704         if (!is_valid_ether_addr(netdev->dev_addr)) {
2705                 eth_hw_addr_random(netdev);
2706                 dev_warn(priv->dev, "using random MAC address %pM\n",
2707                          netdev->dev_addr);
2708                 /* Also copy this new MAC address into hdev */
2709                 if (h->ae_algo->ops->set_mac_addr)
2710                         h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr);
2711         }
2712 }
2713
2714 static void hns3_nic_set_priv_ops(struct net_device *netdev)
2715 {
2716         struct hns3_nic_priv *priv = netdev_priv(netdev);
2717
2718         if ((netdev->features & NETIF_F_TSO) ||
2719             (netdev->features & NETIF_F_TSO6)) {
2720                 priv->ops.fill_desc = hns3_fill_desc_tso;
2721                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
2722         } else {
2723                 priv->ops.fill_desc = hns3_fill_desc;
2724                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
2725         }
2726 }
2727
2728 static int hns3_client_init(struct hnae3_handle *handle)
2729 {
2730         struct pci_dev *pdev = handle->pdev;
2731         struct hns3_nic_priv *priv;
2732         struct net_device *netdev;
2733         int ret;
2734
2735         netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv),
2736                                    handle->kinfo.num_tqps);
2737         if (!netdev)
2738                 return -ENOMEM;
2739
2740         priv = netdev_priv(netdev);
2741         priv->dev = &pdev->dev;
2742         priv->netdev = netdev;
2743         priv->ae_handle = handle;
2744
2745         handle->kinfo.netdev = netdev;
2746         handle->priv = (void *)priv;
2747
2748         hns3_init_mac_addr(netdev);
2749
2750         hns3_set_default_feature(netdev);
2751
2752         netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
2753         netdev->priv_flags |= IFF_UNICAST_FLT;
2754         netdev->netdev_ops = &hns3_nic_netdev_ops;
2755         SET_NETDEV_DEV(netdev, &pdev->dev);
2756         hns3_ethtool_set_ops(netdev);
2757         hns3_nic_set_priv_ops(netdev);
2758
2759         /* Carrier off reporting is important to ethtool even BEFORE open */
2760         netif_carrier_off(netdev);
2761
2762         ret = hns3_get_ring_config(priv);
2763         if (ret) {
2764                 ret = -ENOMEM;
2765                 goto out_get_ring_cfg;
2766         }
2767
2768         ret = hns3_nic_init_vector_data(priv);
2769         if (ret) {
2770                 ret = -ENOMEM;
2771                 goto out_init_vector_data;
2772         }
2773
2774         ret = hns3_init_all_ring(priv);
2775         if (ret) {
2776                 ret = -ENOMEM;
2777                 goto out_init_ring_data;
2778         }
2779
2780         ret = register_netdev(netdev);
2781         if (ret) {
2782                 dev_err(priv->dev, "probe register netdev fail!\n");
2783                 goto out_reg_netdev_fail;
2784         }
2785
2786         /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */
2787         netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
2788
2789         return ret;
2790
2791 out_reg_netdev_fail:
2792 out_init_ring_data:
2793         (void)hns3_nic_uninit_vector_data(priv);
2794         priv->ring_data = NULL;
2795 out_init_vector_data:
2796 out_get_ring_cfg:
2797         priv->ae_handle = NULL;
2798         free_netdev(netdev);
2799         return ret;
2800 }
2801
2802 static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
2803 {
2804         struct net_device *netdev = handle->kinfo.netdev;
2805         struct hns3_nic_priv *priv = netdev_priv(netdev);
2806         int ret;
2807
2808         if (netdev->reg_state != NETREG_UNINITIALIZED)
2809                 unregister_netdev(netdev);
2810
2811         ret = hns3_nic_uninit_vector_data(priv);
2812         if (ret)
2813                 netdev_err(netdev, "uninit vector error\n");
2814
2815         ret = hns3_uninit_all_ring(priv);
2816         if (ret)
2817                 netdev_err(netdev, "uninit ring error\n");
2818
2819         priv->ring_data = NULL;
2820
2821         free_netdev(netdev);
2822 }
2823
2824 static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
2825 {
2826         struct net_device *netdev = handle->kinfo.netdev;
2827
2828         if (!netdev)
2829                 return;
2830
2831         if (linkup) {
2832                 netif_carrier_on(netdev);
2833                 netif_tx_wake_all_queues(netdev);
2834                 netdev_info(netdev, "link up\n");
2835         } else {
2836                 netif_carrier_off(netdev);
2837                 netif_tx_stop_all_queues(netdev);
2838                 netdev_info(netdev, "link down\n");
2839         }
2840 }
2841
2842 const struct hnae3_client_ops client_ops = {
2843         .init_instance = hns3_client_init,
2844         .uninit_instance = hns3_client_uninit,
2845         .link_status_change = hns3_link_status_change,
2846 };
2847
2848 /* hns3_init_module - Driver registration routine
2849  * hns3_init_module is the first routine called when the driver is
2850  * loaded. All it does is register with the PCI subsystem.
2851  */
2852 static int __init hns3_init_module(void)
2853 {
2854         int ret;
2855
2856         pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
2857         pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
2858
2859         client.type = HNAE3_CLIENT_KNIC;
2860         snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
2861                  hns3_driver_name);
2862
2863         client.ops = &client_ops;
2864
2865         ret = hnae3_register_client(&client);
2866         if (ret)
2867                 return ret;
2868
2869         ret = pci_register_driver(&hns3_driver);
2870         if (ret)
2871                 hnae3_unregister_client(&client);
2872
2873         return ret;
2874 }
2875 module_init(hns3_init_module);
2876
2877 /* hns3_exit_module - Driver exit cleanup routine
2878  * hns3_exit_module is called just before the driver is removed
2879  * from memory.
2880  */
2881 static void __exit hns3_exit_module(void)
2882 {
2883         pci_unregister_driver(&hns3_driver);
2884         hnae3_unregister_client(&client);
2885 }
2886 module_exit(hns3_exit_module);
2887
2888 MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
2889 MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
2890 MODULE_LICENSE("GPL");
2891 MODULE_ALIAS("pci:hns-nic");